x
+ diff --git a/server/tests/fixtures/html/extensionless-og-image.html b/server/tests/fixtures/html/extensionless-og-image.html new file mode 100644 index 0000000..0492e00 --- /dev/null +++ b/server/tests/fixtures/html/extensionless-og-image.html @@ -0,0 +1,8 @@ + + + +x
+ diff --git a/server/tests/fixtures/html/h1-fallback.html b/server/tests/fixtures/html/h1-fallback.html new file mode 100644 index 0000000..1936716 --- /dev/null +++ b/server/tests/fixtures/html/h1-fallback.html @@ -0,0 +1,5 @@ + + + +No title element, no meta tags.
+ diff --git a/server/tests/fixtures/html/imgs-missing-src.html b/server/tests/fixtures/html/imgs-missing-src.html new file mode 100644 index 0000000..c522c60 --- /dev/null +++ b/server/tests/fixtures/html/imgs-missing-src.html @@ -0,0 +1,10 @@ + + +x
+ diff --git a/server/tests/fixtures/html/jsonld-graph.html b/server/tests/fixtures/html/jsonld-graph.html new file mode 100644 index 0000000..02562f0 --- /dev/null +++ b/server/tests/fixtures/html/jsonld-graph.html @@ -0,0 +1,15 @@ + + + + + +x
+ diff --git a/server/tests/fixtures/html/jsonld-malformed.html b/server/tests/fixtures/html/jsonld-malformed.html new file mode 100644 index 0000000..eb7e0a8 --- /dev/null +++ b/server/tests/fixtures/html/jsonld-malformed.html @@ -0,0 +1,11 @@ + + + +x
+ diff --git a/server/tests/fixtures/html/keyword-content.html b/server/tests/fixtures/html/keyword-content.html new file mode 100644 index 0000000..2b7f438 --- /dev/null +++ b/server/tests/fixtures/html/keyword-content.html @@ -0,0 +1,16 @@ + + +Beekeeping is rewarding. Beekeeping requires patience. A beehive needs space.
+The beehive must stay dry. Honey harvest comes later. Honey harvest rewards patience.
+Beekeeping beginners should buy a beehive and study honey production carefully.
+Honey production depends on the beehive and on steady beekeeping habits.
+Body text.
x
+ diff --git a/server/tests/fixtures/html/relative-images.html b/server/tests/fixtures/html/relative-images.html new file mode 100644 index 0000000..9739cbb --- /dev/null +++ b/server/tests/fixtures/html/relative-images.html @@ -0,0 +1,9 @@ + + + +x
+ diff --git a/server/tests/fixtures/html/root-relative-icon.html b/server/tests/fixtures/html/root-relative-icon.html new file mode 100644 index 0000000..f6cdc99 --- /dev/null +++ b/server/tests/fixtures/html/root-relative-icon.html @@ -0,0 +1,8 @@ + + + +x
+ diff --git a/server/tests/fixtures/html/sized-images.html b/server/tests/fixtures/html/sized-images.html new file mode 100644 index 0000000..72589f4 --- /dev/null +++ b/server/tests/fixtures/html/sized-images.html @@ -0,0 +1,9 @@ + + +
+ 
+ 
+
+
diff --git a/server/tests/fixtures/html/srcset.html b/server/tests/fixtures/html/srcset.html
new file mode 100644
index 0000000..9f8e368
--- /dev/null
+++ b/server/tests/fixtures/html/srcset.html
@@ -0,0 +1,7 @@
+
+
+x
+ diff --git a/server/tests/fixtures/html/title-only.html b/server/tests/fixtures/html/title-only.html new file mode 100644 index 0000000..b997466 --- /dev/null +++ b/server/tests/fixtures/html/title-only.html @@ -0,0 +1,5 @@ + + +No metadata here at all.
+ diff --git a/server/tests/fixtures/html/tracking-pixels.html b/server/tests/fixtures/html/tracking-pixels.html new file mode 100644 index 0000000..f5c4f2a --- /dev/null +++ b/server/tests/fixtures/html/tracking-pixels.html @@ -0,0 +1,9 @@ + + +
+ 
+ 
+
+
diff --git a/server/tests/fixtures/html/twitter-only.html b/server/tests/fixtures/html/twitter-only.html
new file mode 100644
index 0000000..079a3c5
--- /dev/null
+++ b/server/tests/fixtures/html/twitter-only.html
@@ -0,0 +1,10 @@
+
+
+
+ x
+ diff --git a/server/tests/fixtures/html/webp-og-image.html b/server/tests/fixtures/html/webp-og-image.html new file mode 100644 index 0000000..d36bf7b --- /dev/null +++ b/server/tests/fixtures/html/webp-og-image.html @@ -0,0 +1,8 @@ + + + +x
+ diff --git a/server/tests/helpers/HostQueue.test.js b/server/tests/helpers/HostQueue.test.js new file mode 100644 index 0000000..af673cd --- /dev/null +++ b/server/tests/helpers/HostQueue.test.js @@ -0,0 +1,318 @@ +require('module-alias/register') + +const HostQueue = require('@helpers/HostQueue') + +// +// A controllable clock that also drives the queue's own timers, so advancing +// time is instant and spacing assertions are exact rather than flaky. +// +const makeClock = () => { + let t = 1000 + let nextId = 1 + const timers = new Map() + + return { + now: () => t, + setTimeoutFn: (fn, ms) => { + const id = nextId++ + timers.set(id, { fn, dueAt: t + ms }) + return id + }, + clearTimeoutFn: (id) => { timers.delete(id) }, + advance: (ms) => { + t += ms + for (const [id, timer] of [...timers.entries()]) { + if (timer.dueAt <= t) { + timers.delete(id) + timer.fn() + } + } + }, + } +} + +const wireClock = (clock) => ({ + now: clock.now, + setTimeoutFn: clock.setTimeoutFn, + clearTimeoutFn: clock.clearTimeoutFn, +}) + +const deferred = () => { + let resolve = null + let reject = null + const promise = new Promise((res, rej) => { resolve = res; reject = rej }) + return { promise, resolve, reject } +} + +describe('global concurrency', () => { + + test('never exceeds the cap', async () => { + const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 }) + + let active = 0 + let peak = 0 + const gates = [] + + const jobs = Array.from({ length: 6 }, (unused, i) => { + const gate = deferred() + gates.push(gate) + return queue.schedule(`host${i}.com`, async () => { + active++ + peak = Math.max(peak, active) + await gate.promise + active-- + return i + }) + }) + + // Let the first batch start. + await new Promise(resolve => setImmediate(resolve)) + expect(peak).toBe(2) + + gates.forEach(gate => gate.resolve()) + await Promise.all(jobs) + + expect(peak).toBe(2) + }) + + test('runs different hosts in parallel up to the cap', async () => { + const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 }) + + let active = 0 + let peak = 0 + const gates = [] + + const jobs = ['a.com', 'b.com', 'c.com'].map(host => { + const gate = deferred() + gates.push(gate) + return queue.schedule(host, async () => { + active++ + peak = Math.max(peak, active) + await gate.promise + active-- + }) + }) + + await new Promise(resolve => setImmediate(resolve)) + expect(peak).toBe(3) + + gates.forEach(gate => gate.resolve()) + await Promise.all(jobs) + }) +}) + +describe('per-host serialisation', () => { + + test('runs only one request per host at a time', async () => { + const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 }) + + let active = 0 + let peak = 0 + + const jobs = [1, 2, 3].map(() => queue.schedule('same.com', async () => { + active++ + peak = Math.max(peak, active) + await new Promise(resolve => setImmediate(resolve)) + active-- + })) + + await Promise.all(jobs) + + expect(peak).toBe(1) + }) + + test('preserves FIFO order within a host', async () => { + const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 }) + const order = [] + + await Promise.all([1, 2, 3, 4].map(n => + queue.schedule('same.com', async () => { order.push(n) }) + )) + + expect(order).toEqual([1, 2, 3, 4]) + }) +}) + +describe('politeness spacing', () => { + + test('leaves a gap between two requests to the same host', async () => { + const clock = makeClock() + const queue = new HostQueue.Queue({ + globalConcurrency: 4, + minSpacingMs: 1000, + maxSpacingMs: 1000, + ...wireClock(clock), + }) + + const startedAt = [] + + const first = queue.schedule('same.com', async () => { startedAt.push(clock.now()) }) + await first + + const second = queue.schedule('same.com', async () => { startedAt.push(clock.now()) }) + + // The second job must not have run yet - the host is spaced out. + await new Promise(resolve => setImmediate(resolve)) + expect(startedAt).toHaveLength(1) + + clock.advance(1000) + await second + + expect(startedAt).toHaveLength(2) + expect(startedAt[1] - startedAt[0]).toBeGreaterThanOrEqual(1000) + }) + + test('spacing on one host does not delay a different host', async () => { + const clock = makeClock() + const queue = new HostQueue.Queue({ + globalConcurrency: 4, + minSpacingMs: 5000, + maxSpacingMs: 5000, + ...wireClock(clock), + }) + + await queue.schedule('slow.com', async () => {}) + + // slow.com is now parked for 5s. other.com must be unaffected. + const ran = [] + await queue.schedule('other.com', async () => { ran.push('other') }) + + expect(ran).toEqual(['other']) + }) + + test('honours an explicit spacing override (robots Crawl-delay)', async () => { + const clock = makeClock() + const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0, ...wireClock(clock) }) + + await queue.schedule('slow.com', async () => {}, { spacingMs: 3000 }) + + const ran = [] + const second = queue.schedule('slow.com', async () => { ran.push(1) }) + + await new Promise(resolve => setImmediate(resolve)) + expect(ran).toEqual([]) + + clock.advance(3000) + await second + expect(ran).toEqual([1]) + }) +}) + +describe('error handling', () => { + + test('a rejecting task does not wedge the queue', async () => { + const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 }) + + await expect(queue.schedule('a.com', async () => { throw new Error('boom') })) + .rejects.toThrow('boom') + + await expect(queue.schedule('a.com', async () => 'fine')).resolves.toBe('fine') + }) + + test('a task that throws synchronously is caught', async () => { + const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 }) + + await expect(queue.schedule('a.com', () => { throw new Error('sync boom') })) + .rejects.toThrow('sync boom') + + expect(queue.inflight()).toBe(0) + }) + + test('one failing host does not block others', async () => { + const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 }) + + const results = await Promise.allSettled([ + queue.schedule('bad.com', async () => { throw new Error('nope') }), + queue.schedule('good.com', async () => 'ok'), + ]) + + expect(results[0].status).toBe('rejected') + expect(results[1].value).toBe('ok') + }) +}) + +describe('drain and stop', () => { + + test('drain resolves when everything has finished', async () => { + const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 }) + const done = [] + + ;[1, 2, 3].forEach(n => queue.schedule(`h${n}.com`, async () => { done.push(n) }).catch(() => {})) + + await queue.drain() + + expect(done).toHaveLength(3) + expect(queue.size()).toBe(0) + expect(queue.inflight()).toBe(0) + }) + + test('drain resolves immediately on an idle queue', async () => { + const queue = new HostQueue.Queue() + await expect(queue.drain()).resolves.toBeUndefined() + }) + + test('stop rejects queued work and refuses new work', async () => { + const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 }) + + const gate = deferred() + const first = queue.schedule('a.com', () => gate.promise) + const queued = queue.schedule('b.com', async () => 'never') + + await new Promise(resolve => setImmediate(resolve)) + queue.stop() + + await expect(queued).rejects.toThrow('stopped') + await expect(queue.schedule('c.com', async () => 'x')).rejects.toThrow('stopped') + + gate.resolve('done') + await expect(first).resolves.toBe('done') + }) +}) + +describe('event loop', () => { + + // + // Regression: the politeness timer used to be unref'd, so if nothing else + // held the event loop open Node exited and queued jobs were abandoned with + // their promises never settling. + // + test('a spaced job still resolves with nothing else holding the loop open', async () => { + const queue = new HostQueue.Queue({ minSpacingMs: 25, maxSpacingMs: 25 }) + + await queue.schedule('same.com', async () => 'first') + + // Only the internal timer can resume this one. + await expect(queue.schedule('same.com', async () => 'second')).resolves.toBe('second') + }) + + test('stop() clears the pending timer so shutdown is not blocked', async () => { + const queue = new HostQueue.Queue({ minSpacingMs: 10000, maxSpacingMs: 10000 }) + + await queue.schedule('same.com', async () => 'first') + const queued = queue.schedule('same.com', async () => 'never') + + queue.stop() + + await expect(queued).rejects.toThrow('stopped') + }) +}) + +describe('stats', () => { + + test('reports pending, running and host counts', async () => { + const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 }) + + const gate = deferred() + const first = queue.schedule('a.com', () => gate.promise) + const second = queue.schedule('b.com', async () => {}) + + await new Promise(resolve => setImmediate(resolve)) + + const stats = queue.stats() + expect(stats.running).toBe(1) + expect(stats.pending).toBe(1) + + gate.resolve() + await Promise.all([first, second]) + }) +}) diff --git a/server/tests/helpers/HttpFetch.test.js b/server/tests/helpers/HttpFetch.test.js new file mode 100644 index 0000000..0baf931 --- /dev/null +++ b/server/tests/helpers/HttpFetch.test.js @@ -0,0 +1,479 @@ +require('module-alias/register') + +const HttpFetch = require('@helpers/HttpFetch') +const BrowserProfiles = require('@helpers/BrowserProfiles') +const FetchMock = require('../support/fetchMock') + +const dns = require('dns').promises +let dnsSpy = null + +beforeEach(() => { + FetchMock.install() + // Every hostname in these tests resolves to one public address, so the + // safety gate passes and we are testing HttpFetch rather than DNS. + dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) + // No real wall-time in backoff. + HttpFetch.__setSleep(() => Promise.resolve()) +}) + +afterEach(() => { + FetchMock.restore() + dnsSpy.mockRestore() + HttpFetch.__resetSleep() +}) + +describe('fetchDocument happy path', () => { + + test('returns decoded html, status and final url', async () => { + FetchMock.route('https://example.com/', FetchMock.html('x
', { + headers: { etag: 'W/"abc"', 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT' }, + })) + + const result = await HttpFetch.fetchDocument('https://example.com/') + + expect(result.etag).toBe('W/"abc"') + expect(result.lastModified).toBe('Wed, 01 Jan 2025 00:00:00 GMT') + }) +}) + +describe('fetchDocument browser identity', () => { + + test('sends a complete, coherent header set and never claims to be Googlebot', async () => { + FetchMock.route('https://example.com/', FetchMock.html('x
')) + + await HttpFetch.fetchDocument('https://example.com/') + const headers = FetchMock.calls()[0].headers + + expect(headers['User-Agent']).toMatch(/^Mozilla\/5\.0/) + expect(headers['User-Agent']).not.toMatch(/Googlebot/i) + expect(headers['Accept']).toContain('text/html') + expect(headers['Accept-Language']).toBeTruthy() + expect(headers['Accept-Encoding']).toContain('gzip') + }) + + test('gives the same host the same profile across calls', async () => { + FetchMock.route(/example\.com/, () => FetchMock.html('x
')) + + const first = await HttpFetch.fetchDocument('https://example.com/a') + const second = await HttpFetch.fetchDocument('https://example.com/b') + + expect(first.profileId).toBe(second.profileId) + }) + + test('pairs sec-ch-ua only with Chromium profiles', () => { + const chrome = BrowserProfiles.profileById('chrome-win') + const firefox = BrowserProfiles.profileById('firefox-win') + const safari = BrowserProfiles.profileById('safari-mac') + + const chromeHeaders = BrowserProfiles.documentHeaders(chrome, { url: 'https://a.com/' }) + const firefoxHeaders = BrowserProfiles.documentHeaders(firefox, { url: 'https://a.com/' }) + const safariHeaders = BrowserProfiles.documentHeaders(safari, { url: 'https://a.com/' }) + + expect(chromeHeaders['sec-ch-ua']).toBeTruthy() + expect(chromeHeaders['Sec-Fetch-Mode']).toBe('navigate') + + // Firefox sends Sec-Fetch-* but no client hints. + expect(firefoxHeaders['sec-ch-ua']).toBeUndefined() + expect(firefoxHeaders['Sec-Fetch-Mode']).toBe('navigate') + + // Safari sends neither. + expect(safariHeaders['sec-ch-ua']).toBeUndefined() + expect(safariHeaders['Sec-Fetch-Mode']).toBeUndefined() + }) + + test('never pairs a chrome UA with a firefox accept header', () => { + BrowserProfiles.PROFILES.forEach(profile => { + if (profile.family === 'chrome') { + expect(profile.ua).toMatch(/Chrome\//) + expect(profile.accept).toContain('application/signed-exchange') + } + if (profile.family === 'firefox') { + expect(profile.ua).toMatch(/Firefox\//) + expect(profile.accept).not.toContain('signed-exchange') + } + }) + }) +}) + +describe('fetchDocument redirects', () => { + + test('follows a single hop and reports the final url', async () => { + FetchMock.route('https://example.com/a', FetchMock.redirect('https://example.com/b', 301)) + FetchMock.route('https://example.com/b', () => FetchMock.html('small
', { + headers: { 'content-length': String(50 * 1024 * 1024) }, + })) + + await expect(HttpFetch.fetchDocument('https://example.com/', { retries: 0 })) + .rejects.toMatchObject({ kind: 'too_large' }) + }) + + test('refuses a non-html content type before parsing', async () => { + FetchMock.route('https://example.com/f.pdf', () => FetchMock.buffer(Buffer.alloc(16), { + contentType: 'application/pdf', + })) + + await expect(HttpFetch.fetchDocument('https://example.com/f.pdf', { retries: 0 })) + .rejects.toMatchObject({ kind: 'bad_content_type' }) + }) + + test('refuses a binary masquerading behind an image content type', async () => { + FetchMock.route('https://example.com/x', () => FetchMock.buffer(Buffer.alloc(16), { + contentType: 'image/png', + })) + + await expect(HttpFetch.fetchDocument('https://example.com/x', { retries: 0 })) + .rejects.toMatchObject({ kind: 'bad_content_type' }) + }) +}) + +describe('fetchDocument charset handling', () => { + + test('decodes windows-1252 declared in the content-type header', async () => { + // 0x93/0x94 are curly quotes in cp1252 and invalid in utf-8. + const body = Buffer.concat([ + Buffer.from('ok
'), { + contentType: 'text/html; charset=not-a-real-charset', + })) + + const result = await HttpFetch.fetchDocument('https://example.com/') + expect(result.html).toContain('ok') + }) +}) + +describe('fetchDocument conditional requests', () => { + + test('sends If-None-Match and If-Modified-Since when given', async () => { + FetchMock.route('https://example.com/', () => FetchMock.html('x
')) + + await HttpFetch.fetchDocument('https://example.com/', { + conditional: { etag: 'W/"v1"', lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT' }, + }) + + const headers = FetchMock.calls()[0].headers + expect(headers['If-None-Match']).toBe('W/"v1"') + expect(headers['If-Modified-Since']).toBe('Wed, 01 Jan 2025 00:00:00 GMT') + }) + + test('reports a 304 as notModified with no body', async () => { + FetchMock.route('https://example.com/', () => FetchMock.status(304)) + + const result = await HttpFetch.fetchDocument('https://example.com/', { + conditional: { etag: 'W/"v1"' }, + }) + + expect(result.notModified).toBe(true) + expect(result.status).toBe(304) + expect(result.html).toBeNull() + }) +}) + +describe('fetchBinary', () => { + + const pngMagic = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + + test('returns the buffer for a real image', async () => { + FetchMock.route('https://example.com/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' })) + + const result = await HttpFetch.fetchBinary('https://example.com/i.png') + + expect(result.mime).toBe('image/png') + expect(result.buffer.equals(pngMagic)).toBe(true) + }) + + // The old code wrote a 404 HTML error page to disk and handed it to gm. + test('refuses an html error page served where an image was expected', async () => { + FetchMock.route('https://example.com/i.png', () => FetchMock.html('x
')) + + await expect(HttpFetch.fetchDocument('http://10.0.0.1/', { retries: 0 })) + .rejects.toMatchObject({ kind: 'blocked', reason: 'private_ip' }) + }) + + test('classifies a DNS failure as network, not blocked', async () => { + dnsSpy.mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' })) + FetchMock.route(/.*/, () => FetchMock.html('x
')) + + await expect(HttpFetch.fetchDocument('https://gone.example/', { retries: 0 })) + .rejects.toMatchObject({ kind: 'network', reason: 'dns_failed' }) + }) + + test('retries a DNS failure, since it may be transient', async () => { + let attempts = 0 + dnsSpy.mockImplementation(async () => { + attempts++ + if (attempts < 2) { throw Object.assign(new Error('nope'), { code: 'ENOTFOUND' }) } + return [{ address: '93.184.216.34', family: 4 }] + }) + FetchMock.route('https://flaky.example/', () => FetchMock.html('x
')) + + await expect(HttpFetch.fetchDocument('http://127.0.0.1/')).rejects.toMatchObject({ kind: 'blocked' }) + expect(FetchMock.callCount()).toBe(0) + }) +}) + +describe('safety gate on the initial url', () => { + + test('refuses to open a connection to a private address at all', async () => { + FetchMock.route(/.*/, () => FetchMock.html('should never be reached
')) + + await expect(HttpFetch.fetchDocument('http://127.0.0.1:3000/', { retries: 0 })) + .rejects.toMatchObject({ kind: 'blocked' }) + + expect(FetchMock.callCount()).toBe(0) + }) +}) diff --git a/server/tests/helpers/Robots.test.js b/server/tests/helpers/Robots.test.js new file mode 100644 index 0000000..93f31dd --- /dev/null +++ b/server/tests/helpers/Robots.test.js @@ -0,0 +1,173 @@ +require('module-alias/register') + +const Robots = require('@helpers/Robots') +const config = require('@config/scrape') +const FetchMock = require('../support/fetchMock') + +const dns = require('dns').promises +let dnsSpy = null + +const UA = config.robotsUaToken + +beforeEach(() => { + FetchMock.install() + Robots.__resetCache() + dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) +}) + +afterEach(() => { + FetchMock.restore() + dnsSpy.mockRestore() + Robots.__resetCache() +}) + +const serveRobots = (body, origin = 'https://example.com') => { + FetchMock.route(`${origin}/robots.txt`, () => new Response(body, { + status: 200, + headers: { 'content-type': 'text/plain' }, + })) +} + +describe('Robots.parse', () => { + + test('groups consecutive user-agent lines into one rule block', () => { + const rules = Robots.parse([ + 'User-agent: alpha', + 'User-agent: beta', + 'Disallow: /private', + ].join('\n')) + + expect(rules.groups).toHaveLength(1) + expect(rules.groups[0].agents).toEqual(['alpha', 'beta']) + expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/private' }]) + }) + + test('ignores comments and blank lines', () => { + const rules = Robots.parse('# a comment\n\nUser-agent: *\nDisallow: /x # trailing\n') + expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/x' }]) + }) + + test('treats an empty Disallow as no restriction', () => { + const rules = Robots.parse('User-agent: *\nDisallow:') + expect(rules.groups[0].rules).toHaveLength(0) + }) + + test('parses crawl-delay', () => { + const rules = Robots.parse('User-agent: *\nCrawl-delay: 2.5') + expect(rules.groups[0].crawlDelayMs).toBe(2500) + }) + + test('an empty file allows everything', () => { + expect(Robots.parse('').allowAll).toBe(true) + expect(Robots.parse(null).allowAll).toBe(true) + }) +}) + +describe('Robots.isPathAllowed', () => { + + test('respects a simple disallow', () => { + const rules = Robots.parse('User-agent: *\nDisallow: /private') + expect(Robots.isPathAllowed(rules, '/private/x', UA)).toBe(false) + expect(Robots.isPathAllowed(rules, '/public/x', UA)).toBe(true) + }) + + test('longest match wins, so a nested Allow overrides a broad Disallow', () => { + const rules = Robots.parse('User-agent: *\nDisallow: /a\nAllow: /a/b') + expect(Robots.isPathAllowed(rules, '/a/x', UA)).toBe(false) + expect(Robots.isPathAllowed(rules, '/a/b/c', UA)).toBe(true) + }) + + test('Allow beats Disallow at equal specificity', () => { + const rules = Robots.parse('User-agent: *\nDisallow: /p\nAllow: /p') + expect(Robots.isPathAllowed(rules, '/p', UA)).toBe(true) + }) + + test('supports the * wildcard', () => { + const rules = Robots.parse('User-agent: *\nDisallow: /*.pdf') + expect(Robots.isPathAllowed(rules, '/docs/report.pdf', UA)).toBe(false) + expect(Robots.isPathAllowed(rules, '/docs/report.html', UA)).toBe(true) + }) + + test('supports the $ anchor', () => { + const rules = Robots.parse('User-agent: *\nDisallow: /x$') + expect(Robots.isPathAllowed(rules, '/x', UA)).toBe(false) + expect(Robots.isPathAllowed(rules, '/x/y', UA)).toBe(true) + }) + + test('a named group beats the wildcard group even when listed later', () => { + const rules = Robots.parse([ + 'User-agent: *', + 'Disallow: /', + '', + `User-agent: ${UA}`, + 'Disallow:', + ].join('\n')) + + expect(Robots.isPathAllowed(rules, '/anything', UA)).toBe(true) + expect(Robots.isPathAllowed(rules, '/anything', 'SomeOtherBot')).toBe(false) + }) +}) + +describe('Robots.isAllowed over the network', () => { + + test('refuses a disallowed path', async () => { + serveRobots('User-agent: *\nDisallow: /secret') + await expect(Robots.isAllowed('https://example.com/secret/page')).resolves.toBe(false) + }) + + test('permits an allowed path', async () => { + serveRobots('User-agent: *\nDisallow: /secret') + await expect(Robots.isAllowed('https://example.com/blog/post')).resolves.toBe(true) + }) + + // Fail-open: a missing robots.txt is not a refusal. + test('allows everything when robots.txt 404s', async () => { + FetchMock.route('https://example.com/robots.txt', () => FetchMock.status(404)) + await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true) + }) + + test('allows everything when robots.txt cannot be reached', async () => { + FetchMock.route('https://example.com/robots.txt', FetchMock.networkError()) + await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true) + }) + + test('caches per origin - two queries, one network call', async () => { + serveRobots('User-agent: *\nDisallow: /secret') + + await Robots.isAllowed('https://example.com/a') + await Robots.isAllowed('https://example.com/secret/b') + + expect(FetchMock.callsTo('https://example.com/robots.txt')).toHaveLength(1) + }) + + test('keeps origins separate', async () => { + serveRobots('User-agent: *\nDisallow: /', 'https://blocked.com') + serveRobots('User-agent: *\nDisallow:', 'https://open.com') + + await expect(Robots.isAllowed('https://blocked.com/x')).resolves.toBe(false) + await expect(Robots.isAllowed('https://open.com/x')).resolves.toBe(true) + }) + + test('returns false for an unparseable url rather than throwing', async () => { + await expect(Robots.isAllowed('not a url')).resolves.toBe(false) + }) +}) + +describe('Robots.crawlDelayMs', () => { + + test('reports the declared delay', async () => { + serveRobots('User-agent: *\nCrawl-delay: 5') + await expect(Robots.crawlDelayMs('https://example.com/x')).resolves.toBe(5000) + }) + + test('is null when none is declared', async () => { + serveRobots('User-agent: *\nDisallow: /x') + await expect(Robots.crawlDelayMs('https://example.com/y')).resolves.toBeNull() + }) + + test('flags a delay beyond the cap instead of parking the job forever', async () => { + expect(Robots.exceedsCrawlDelayCap(3600 * 1000)).toBe(true) + expect(Robots.exceedsCrawlDelayCap(2000)).toBe(false) + expect(Robots.exceedsCrawlDelayCap(null)).toBe(false) + }) +}) diff --git a/server/tests/helpers/SiteScrape.test.js b/server/tests/helpers/SiteScrape.test.js new file mode 100644 index 0000000..c01c4ab --- /dev/null +++ b/server/tests/helpers/SiteScrape.test.js @@ -0,0 +1,318 @@ +require('module-alias/register') + +const fs = require('fs') +const path = require('path') +const cheerio = require('cheerio') + +const SiteScrape = require('@helpers/SiteScrape') + +const FIXTURES = path.join(__dirname, '..', 'fixtures', 'html') + +// Fixtures are frozen snapshots - see fixtures/html/README.md. No network here. +const load = (name) => cheerio.load(fs.readFileSync(path.join(FIXTURES, name), 'utf8')) + +const PAGE_URL = 'https://example.com/blog/2026/post' + +describe('getTitle', () => { + + test('prefers og:title over the title element', () => { + expect(SiteScrape.getTitle(load('og-complete.html'))).toBe('The Real Title') + }) + + test('falls back to twitter:title', () => { + expect(SiteScrape.getTitle(load('twitter-only.html'))).toBe('Twitter Card Title') + }) + + test('falls back to JSON-LD headline', () => { + expect(SiteScrape.getTitle(load('jsonld-article.html'))).toBe('JSON-LD Headline') + }) + + test('falls back to the title element and collapses whitespace', () => { + expect(SiteScrape.getTitle(load('title-only.html'))).toBe('Just A Title') + }) + + test('falls back to an h1 when there is no title element', () => { + expect(SiteScrape.getTitle(load('h1-fallback.html'))).toBe('Heading Is All We Have') + }) + + test('returns empty string for a page with nothing', () => { + expect(SiteScrape.getTitle(load('no-metadata.html'))).toBe('') + }) +}) + +describe('getDescription', () => { + + test('prefers og:description', () => { + expect(SiteScrape.getDescription(load('og-complete.html'))) + .toBe('A proper social description of the page.') + }) + + test('falls back to twitter:description', () => { + expect(SiteScrape.getDescription(load('twitter-only.html'))).toBe('Twitter card description.') + }) + + test('falls back to JSON-LD description', () => { + expect(SiteScrape.getDescription(load('jsonld-article.html'))).toBe('Described in structured data.') + }) + + test('is empty when no description exists', () => { + expect(SiteScrape.getDescription(load('title-only.html'))).toBe('') + }) +}) + +describe('getJsonLd', () => { + + test('reads a plain block', () => { + expect(SiteScrape.getJsonLd(load('jsonld-article.html')).headline).toBe('JSON-LD Headline') + }) + + test('digs the content node out of an @graph', () => { + expect(SiteScrape.getJsonLd(load('jsonld-graph.html')).headline).toBe('Graph Headline') + }) + + // Malformed JSON-LD is extremely common and must never throw. + test('survives malformed JSON without throwing', () => { + expect(() => SiteScrape.getJsonLd(load('jsonld-malformed.html'))).not.toThrow() + expect(SiteScrape.getTitle(load('jsonld-malformed.html'))).toBe('Survived Bad JSON') + }) + + test('is null when absent', () => { + expect(SiteScrape.getJsonLd(load('title-only.html'))).toBeNull() + }) +}) + +describe('getCanonicalUrl', () => { + + test('uses link rel=canonical', () => { + expect(SiteScrape.getCanonicalUrl(load('og-complete.html'), PAGE_URL)) + .toBe('https://example.com/canonical-post') + }) + + test('falls back to the requested url', () => { + expect(SiteScrape.getCanonicalUrl(load('title-only.html'), PAGE_URL)).toBe(PAGE_URL) + }) +}) + +describe('getDisplayImage', () => { + + test('prefers og:image', () => { + expect(SiteScrape.getDisplayImage(load('og-complete.html'), PAGE_URL)) + .toBe('https://cdn.example.com/hero.jpg') + }) + + test('falls back to twitter:image', () => { + expect(SiteScrape.getDisplayImage(load('twitter-only.html'), PAGE_URL)) + .toBe('https://cdn.example.com/twitter.png') + }) + + test('falls back to the JSON-LD image, taking the first of an array', () => { + expect(SiteScrape.getDisplayImage(load('jsonld-article.html'), PAGE_URL)) + .toBe('https://cdn.example.com/ld-primary.jpg') + }) + + test('reads an object-form JSON-LD image', () => { + expect(SiteScrape.getDisplayImage(load('jsonld-graph.html'), PAGE_URL)) + .toBe('https://cdn.example.com/graph.png') + }) + + // Resolution against the canonical base, not string concatenation. + test('resolves a relative path against the canonical url', () => { + expect(SiteScrape.getDisplayImage(load('relative-images.html'), PAGE_URL)) + .toBe('https://example.com/blog/images/hero.png') + }) + + test('resolves a root-relative icon', () => { + expect(SiteScrape.getDisplayImage(load('root-relative-icon.html'), PAGE_URL)) + .toBe('https://example.com/favicon.ico') + }) + + test('resolves a protocol-relative url', () => { + expect(SiteScrape.getDisplayImage(load('protocol-relative-image.html'), PAGE_URL)) + .toBe('https://cdn.example.com/proto.jpg') + }) + + // The old substring filter dropped all of these. + test('accepts .webp', () => { + expect(SiteScrape.getDisplayImage(load('webp-og-image.html'), PAGE_URL)) + .toBe('https://cdn.example.com/hero.webp') + }) + + test('accepts an extensionless og:image endpoint', () => { + expect(SiteScrape.getDisplayImage(load('extensionless-og-image.html'), PAGE_URL)) + .toBe('https://example.com/api/og?title=Hello&theme=dark') + }) + + test('rejects svg while allowSvgThumbnails is off', () => { + expect(SiteScrape.getDisplayImage(load('svg-only-image.html'), PAGE_URL)).toBe('') + }) + + test('picks the largest srcset candidate', () => { + expect(SiteScrape.getDisplayImage(load('srcset.html'), PAGE_URL)) + .toBe('https://cdn.example.com/large.jpg') + }) + + test('prefers the largest inline image by declared area', () => { + expect(SiteScrape.getDisplayImage(load('sized-images.html'), PAGE_URL)) + .toBe('https://cdn.example.com/big.jpg') + }) + + test('skips tracking pixels and spacers', () => { + expect(SiteScrape.getDisplayImage(load('tracking-pixels.html'), PAGE_URL)) + .toBe('https://cdn.example.com/real-photo.jpg') + }) + + // Regression: the old code read element.attribs.src directly and threw. + test('does not throw on img tags with no src, and finds a data-src', () => { + const $ = load('imgs-missing-src.html') + expect(() => SiteScrape.getDisplayImage($, PAGE_URL)).not.toThrow() + expect(SiteScrape.getDisplayImage($, PAGE_URL)).toBe('https://cdn.example.com/lazy.jpg') + }) + + test('prefers a large apple-touch-icon over a 16x16 favicon', () => { + expect(SiteScrape.getDisplayImage(load('apple-touch-icon.html'), PAGE_URL)) + .toBe('https://example.com/touch-180.png') + }) + + test('returns empty string when there is no usable image', () => { + expect(SiteScrape.getDisplayImage(load('no-metadata.html'), PAGE_URL)).toBe('') + }) + + test('survives malformed markup', () => { + expect(() => SiteScrape.getDisplayImage(load('malformed-unclosed.html'), PAGE_URL)).not.toThrow() + }) +}) + +describe('getImageCandidates', () => { + + test('returns an ordered, deduplicated list', () => { + const candidates = SiteScrape.getImageCandidates(load('tracking-pixels.html'), PAGE_URL) + expect(candidates).toEqual(['https://cdn.example.com/real-photo.jpg']) + }) + + test('is empty rather than null for a bare page', () => { + expect(SiteScrape.getImageCandidates(load('no-metadata.html'), PAGE_URL)).toEqual([]) + }) +}) + +describe('getKeywords', () => { + + test('extracts page terms and ignores nav, script and footer chrome', () => { + const keywords = SiteScrape.getKeywords(load('keyword-content.html')) + + expect(keywords).toMatch(/^Keywords: /) + expect(keywords).toMatch(/beekeeping|beehive|honey/) + expect(keywords).not.toMatch(/navigation/) + expect(keywords).not.toMatch(/tracking/) + expect(keywords).not.toMatch(/footer/) + }) + + test('returns empty string when there is nothing to count', () => { + expect(SiteScrape.getKeywords(load('no-metadata.html'))).toBe('') + }) + + test('does not log page content', () => { + // The old implementation dumped up to 6000 chars of every page. + const spy = jest.spyOn(console, 'log').mockImplementation(() => {}) + SiteScrape.getKeywords(load('keyword-content.html')) + expect(spy).not.toHaveBeenCalled() + spy.mockRestore() + }) +}) + +describe('getHostName', () => { + + test('returns the https origin', () => { + expect(SiteScrape.getHostName('http://www.google.com/search?q=x')).toBe('https://www.google.com') + }) + + test('returns empty string for a bad url instead of throwing', () => { + expect(() => SiteScrape.getHostName('nonsense')).not.toThrow() + expect(SiteScrape.getHostName('nonsense')).toBe('') + }) +}) + +describe('buildSearchText', () => { + + test('joins the parts that exist', () => { + const text = SiteScrape.buildSearchText({ + title: 'T', description: 'D', siteName: 'S', keywords: 'Keywords: a, b', + }) + expect(text).toBe('T\nD\nS\nKeywords: a, b') + }) + + test('skips empty parts', () => { + expect(SiteScrape.buildSearchText({ title: 'T', description: '', siteName: '', keywords: '' })).toBe('T') + }) + + test('caps the length so it cannot bloat attachment.text', () => { + const text = SiteScrape.buildSearchText({ title: 'x'.repeat(50000) }) + expect(text.length).toBeLessThanOrEqual(8000) + }) + + // Sentinel strings used to land in attachment.text and pollute FULLTEXT. + test('never produces the old failure sentinels', () => { + const text = SiteScrape.buildSearchText({ title: '', description: '', siteName: '', keywords: '' }) + expect(text).toBe('') + expect(text).not.toContain('No site text') + expect(text).not.toContain('Request Timeout') + }) +}) + +describe('parse', () => { + + test('returns everything a link preview needs in one call', () => { + const result = SiteScrape.parse(load('og-complete.html'), PAGE_URL) + + expect(result).toMatchObject({ + title: 'The Real Title', + description: 'A proper social description of the page.', + siteName: 'Example Journal', + canonicalUrl: 'https://example.com/canonical-post', + imageUrl: 'https://cdn.example.com/hero.jpg', + hostname: 'https://example.com', + }) + expect(result.searchText).toContain('The Real Title') + expect(result.searchText).toContain('A proper social description') + }) + + test('degrades to empty fields rather than throwing on a bare page', () => { + const result = SiteScrape.parse(load('no-metadata.html'), PAGE_URL) + + expect(result.title).toBe('') + expect(result.imageUrl).toBe('') + expect(result.searchText).toBe('example.com') + }) +}) + +// +// getCleanUrls is deliberately unchanged - it decides which URLs are detected in +// note text and what string is stored in attachment.url. These tests pin the +// CURRENT behaviour so the rewrite around it cannot drift link detection. +// +describe('getCleanUrls (pinned, must not change)', () => { + + test('finds bare and full urls', () => { + expect(SiteScrape.getCleanUrls('see https://example.com/a and www.foo.com here')) + .toEqual(['https://example.com/a', 'https://www.foo.com']) + }) + + test('still forces http to https', () => { + expect(SiteScrape.getCleanUrls('http://example.com/x')).toEqual(['https://example.com/x']) + }) + + test('still prepends https to a bare www host', () => { + expect(SiteScrape.getCleanUrls('www.example.com')).toEqual(['https://www.example.com']) + }) + + test('deduplicates', () => { + expect(SiteScrape.getCleanUrls('https://a.com https://a.com')).toEqual(['https://a.com']) + }) + + test('returns an empty array when there are no urls', () => { + expect(SiteScrape.getCleanUrls('just some words')).toEqual([]) + }) + + test('returns an empty array for empty text', () => { + expect(SiteScrape.getCleanUrls('')).toEqual([]) + }) +}) diff --git a/server/tests/helpers/UrlSafety.test.js b/server/tests/helpers/UrlSafety.test.js new file mode 100644 index 0000000..dbdcc6f --- /dev/null +++ b/server/tests/helpers/UrlSafety.test.js @@ -0,0 +1,189 @@ +require('module-alias/register') + +const UrlSafety = require('@helpers/UrlSafety') + +describe('UrlSafety.isPrivateIp', () => { + + const privateAddresses = [ + '0.0.0.0', '10.0.0.1', '10.255.255.254', '127.0.0.1', '127.1.1.1', + '169.254.1.1', '169.254.169.254', // cloud metadata endpoint + '172.16.0.1', '172.31.255.254', '192.168.1.1', '192.0.0.1', '192.0.2.5', + '198.18.0.1', '100.64.0.1', '100.127.255.254', '224.0.0.1', '255.255.255.255', + '::1', '::', 'fc00::1', 'fd12:3456::1', 'fe80::1', 'ff02::1', + '::ffff:127.0.0.1', '::ffff:10.0.0.1', '64:ff9b::127.0.0.1', + ] + + privateAddresses.forEach(address => { + test(`blocks ${address}`, () => { + expect(UrlSafety.isPrivateIp(address)).toBe(true) + }) + }) + + const publicAddresses = ['1.1.1.1', '8.8.8.8', '93.184.216.34', '172.32.0.1', '2606:4700::1111'] + + publicAddresses.forEach(address => { + test(`allows ${address}`, () => { + expect(UrlSafety.isPrivateIp(address)).toBe(false) + }) + }) + + test('treats a non-IP string as unsafe', () => { + expect(UrlSafety.isPrivateIp('not-an-ip')).toBe(true) + expect(UrlSafety.isPrivateIp('')).toBe(true) + }) +}) + +describe('UrlSafety.parse', () => { + + test('rejects non-http schemes', () => { + expect(UrlSafety.parse('file:///etc/passwd')).toBeNull() + expect(UrlSafety.parse('ftp://example.com/x')).toBeNull() + expect(UrlSafety.parse('javascript:alert(1)')).toBeNull() + expect(UrlSafety.parse('data:text/html,body
+`) + +// Clear cache rows for a URL so each test starts from a real miss. +const clearCache = (url) => db.promise() + .query('DELETE FROM scrape_cache WHERE url_hash = ?', [UrlSafety.urlKey(url)]) + +const attachmentRow = (id) => db.promise() + .query('SELECT * FROM attachment WHERE id = ?', [id]) + .then(([rows]) => rows[0] || null) + +beforeAll(async () => { + await Schema.ensureScrapeSchema() + + try { + const existing = await User.getByUserName(testUserName) + if (existing && existing.id) { await User.deleteUser(existing.id, password) } + } catch (error) { /* first run */ } + + const registered = await User.register(testUserName, password) + userId = registered.userId +}) + +beforeEach(() => { + FetchMock.install() + Robots.__resetCache() + dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) + HttpFetch.__setSleep(() => Promise.resolve()) + + // Exercise the real code path with politeness spacing and robots disabled. + ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false }) +}) + +afterEach(() => { + FetchMock.restore() + dnsSpy.mockRestore() + HttpFetch.__resetSleep() + ScrapeQueue.__reset() +}) + +afterAll(async () => { + await db.promise().query('DELETE FROM attachment WHERE user_id = ?', [userId]) + db.end() +}) + +describe('fetch and persist', () => { + + test('writes parsed metadata onto the attachment row as plaintext', async () => { + const url = 'https://example.com/post-one' + await clearCache(url) + FetchMock.route(url, () => page('Post One', 'the first post')) + + const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url) + await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url }) + + const row = await attachmentRow(attachmentId) + + expect(row.text).toContain('Post One') + expect(row.text).toContain('the first post') + expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE) + // Must stay unencrypted - the FULLTEXT index depends on it. + expect(row.text).not.toMatch(/^[0-9a-f]{64}/) + }) + + test('keeps the url the user wrote rather than the canonical url', async () => { + const url = 'https://example.com/original' + await clearCache(url) + + FetchMock.route(url, () => FetchMock.html(` +
+
+
diff --git a/server/tests/fixtures/html/jsonld-article.html b/server/tests/fixtures/html/jsonld-article.html
new file mode 100644
index 0000000..088e51c
--- /dev/null
+++ b/server/tests/fixtures/html/jsonld-article.html
@@ -0,0 +1,15 @@
+
+
+
+
+
+
+
+
diff --git a/server/tests/fixtures/html/svg-only-image.html b/server/tests/fixtures/html/svg-only-image.html
new file mode 100644
index 0000000..31657c5
--- /dev/null
+++ b/server/tests/fixtures/html/svg-only-image.html
@@ -0,0 +1,8 @@
+
+
+
+