Adding scrape files for testing
This commit is contained in:
Max committed
1 parent
7d255fe687
commit
8d60aae65c
40 files changed
+4423
No files matched your search
@@ -0,0 +1,318 @@
|
||||
require('module-alias/register')
|
||||
|
||||
const HostQueue = require('@helpers/HostQueue')
|
||||
|
||||
//
|
||||
// A controllable clock that also drives the queue's own timers, so advancing
|
||||
// time is instant and spacing assertions are exact rather than flaky.
|
||||
//
|
||||
const makeClock = () => {
|
||||
let t = 1000
|
||||
let nextId = 1
|
||||
const timers = new Map()
|
||||
|
||||
return {
|
||||
now: () => t,
|
||||
setTimeoutFn: (fn, ms) => {
|
||||
const id = nextId++
|
||||
timers.set(id, { fn, dueAt: t + ms })
|
||||
return id
|
||||
},
|
||||
clearTimeoutFn: (id) => { timers.delete(id) },
|
||||
advance: (ms) => {
|
||||
t += ms
|
||||
for (const [id, timer] of [...timers.entries()]) {
|
||||
if (timer.dueAt <= t) {
|
||||
timers.delete(id)
|
||||
timer.fn()
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const wireClock = (clock) => ({
|
||||
now: clock.now,
|
||||
setTimeoutFn: clock.setTimeoutFn,
|
||||
clearTimeoutFn: clock.clearTimeoutFn,
|
||||
})
|
||||
|
||||
const deferred = () => {
|
||||
let resolve = null
|
||||
let reject = null
|
||||
const promise = new Promise((res, rej) => { resolve = res; reject = rej })
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
|
||||
describe('global concurrency', () => {
|
||||
|
||||
test('never exceeds the cap', async () => {
|
||||
const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const gates = []
|
||||
|
||||
const jobs = Array.from({ length: 6 }, (unused, i) => {
|
||||
const gate = deferred()
|
||||
gates.push(gate)
|
||||
return queue.schedule(`host${i}.com`, async () => {
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
await gate.promise
|
||||
active--
|
||||
return i
|
||||
})
|
||||
})
|
||||
|
||||
// Let the first batch start.
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
expect(peak).toBe(2)
|
||||
|
||||
gates.forEach(gate => gate.resolve())
|
||||
await Promise.all(jobs)
|
||||
|
||||
expect(peak).toBe(2)
|
||||
})
|
||||
|
||||
test('runs different hosts in parallel up to the cap', async () => {
|
||||
const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const gates = []
|
||||
|
||||
const jobs = ['a.com', 'b.com', 'c.com'].map(host => {
|
||||
const gate = deferred()
|
||||
gates.push(gate)
|
||||
return queue.schedule(host, async () => {
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
await gate.promise
|
||||
active--
|
||||
})
|
||||
})
|
||||
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
expect(peak).toBe(3)
|
||||
|
||||
gates.forEach(gate => gate.resolve())
|
||||
await Promise.all(jobs)
|
||||
})
|
||||
})
|
||||
|
||||
describe('per-host serialisation', () => {
|
||||
|
||||
test('runs only one request per host at a time', async () => {
|
||||
const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
|
||||
let active = 0
|
||||
let peak = 0
|
||||
|
||||
const jobs = [1, 2, 3].map(() => queue.schedule('same.com', async () => {
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
active--
|
||||
}))
|
||||
|
||||
await Promise.all(jobs)
|
||||
|
||||
expect(peak).toBe(1)
|
||||
})
|
||||
|
||||
test('preserves FIFO order within a host', async () => {
|
||||
const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
const order = []
|
||||
|
||||
await Promise.all([1, 2, 3, 4].map(n =>
|
||||
queue.schedule('same.com', async () => { order.push(n) })
|
||||
))
|
||||
|
||||
expect(order).toEqual([1, 2, 3, 4])
|
||||
})
|
||||
})
|
||||
|
||||
describe('politeness spacing', () => {
|
||||
|
||||
test('leaves a gap between two requests to the same host', async () => {
|
||||
const clock = makeClock()
|
||||
const queue = new HostQueue.Queue({
|
||||
globalConcurrency: 4,
|
||||
minSpacingMs: 1000,
|
||||
maxSpacingMs: 1000,
|
||||
...wireClock(clock),
|
||||
})
|
||||
|
||||
const startedAt = []
|
||||
|
||||
const first = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
|
||||
await first
|
||||
|
||||
const second = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
|
||||
|
||||
// The second job must not have run yet - the host is spaced out.
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
expect(startedAt).toHaveLength(1)
|
||||
|
||||
clock.advance(1000)
|
||||
await second
|
||||
|
||||
expect(startedAt).toHaveLength(2)
|
||||
expect(startedAt[1] - startedAt[0]).toBeGreaterThanOrEqual(1000)
|
||||
})
|
||||
|
||||
test('spacing on one host does not delay a different host', async () => {
|
||||
const clock = makeClock()
|
||||
const queue = new HostQueue.Queue({
|
||||
globalConcurrency: 4,
|
||||
minSpacingMs: 5000,
|
||||
maxSpacingMs: 5000,
|
||||
...wireClock(clock),
|
||||
})
|
||||
|
||||
await queue.schedule('slow.com', async () => {})
|
||||
|
||||
// slow.com is now parked for 5s. other.com must be unaffected.
|
||||
const ran = []
|
||||
await queue.schedule('other.com', async () => { ran.push('other') })
|
||||
|
||||
expect(ran).toEqual(['other'])
|
||||
})
|
||||
|
||||
test('honours an explicit spacing override (robots Crawl-delay)', async () => {
|
||||
const clock = makeClock()
|
||||
const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0, ...wireClock(clock) })
|
||||
|
||||
await queue.schedule('slow.com', async () => {}, { spacingMs: 3000 })
|
||||
|
||||
const ran = []
|
||||
const second = queue.schedule('slow.com', async () => { ran.push(1) })
|
||||
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
expect(ran).toEqual([])
|
||||
|
||||
clock.advance(3000)
|
||||
await second
|
||||
expect(ran).toEqual([1])
|
||||
})
|
||||
})
|
||||
|
||||
describe('error handling', () => {
|
||||
|
||||
test('a rejecting task does not wedge the queue', async () => {
|
||||
const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
|
||||
await expect(queue.schedule('a.com', async () => { throw new Error('boom') }))
|
||||
.rejects.toThrow('boom')
|
||||
|
||||
await expect(queue.schedule('a.com', async () => 'fine')).resolves.toBe('fine')
|
||||
})
|
||||
|
||||
test('a task that throws synchronously is caught', async () => {
|
||||
const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
|
||||
await expect(queue.schedule('a.com', () => { throw new Error('sync boom') }))
|
||||
.rejects.toThrow('sync boom')
|
||||
|
||||
expect(queue.inflight()).toBe(0)
|
||||
})
|
||||
|
||||
test('one failing host does not block others', async () => {
|
||||
const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
|
||||
const results = await Promise.allSettled([
|
||||
queue.schedule('bad.com', async () => { throw new Error('nope') }),
|
||||
queue.schedule('good.com', async () => 'ok'),
|
||||
])
|
||||
|
||||
expect(results[0].status).toBe('rejected')
|
||||
expect(results[1].value).toBe('ok')
|
||||
})
|
||||
})
|
||||
|
||||
describe('drain and stop', () => {
|
||||
|
||||
test('drain resolves when everything has finished', async () => {
|
||||
const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
const done = []
|
||||
|
||||
;[1, 2, 3].forEach(n => queue.schedule(`h${n}.com`, async () => { done.push(n) }).catch(() => {}))
|
||||
|
||||
await queue.drain()
|
||||
|
||||
expect(done).toHaveLength(3)
|
||||
expect(queue.size()).toBe(0)
|
||||
expect(queue.inflight()).toBe(0)
|
||||
})
|
||||
|
||||
test('drain resolves immediately on an idle queue', async () => {
|
||||
const queue = new HostQueue.Queue()
|
||||
await expect(queue.drain()).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
test('stop rejects queued work and refuses new work', async () => {
|
||||
const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
|
||||
const gate = deferred()
|
||||
const first = queue.schedule('a.com', () => gate.promise)
|
||||
const queued = queue.schedule('b.com', async () => 'never')
|
||||
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
queue.stop()
|
||||
|
||||
await expect(queued).rejects.toThrow('stopped')
|
||||
await expect(queue.schedule('c.com', async () => 'x')).rejects.toThrow('stopped')
|
||||
|
||||
gate.resolve('done')
|
||||
await expect(first).resolves.toBe('done')
|
||||
})
|
||||
})
|
||||
|
||||
describe('event loop', () => {
|
||||
|
||||
//
|
||||
// Regression: the politeness timer used to be unref'd, so if nothing else
|
||||
// held the event loop open Node exited and queued jobs were abandoned with
|
||||
// their promises never settling.
|
||||
//
|
||||
test('a spaced job still resolves with nothing else holding the loop open', async () => {
|
||||
const queue = new HostQueue.Queue({ minSpacingMs: 25, maxSpacingMs: 25 })
|
||||
|
||||
await queue.schedule('same.com', async () => 'first')
|
||||
|
||||
// Only the internal timer can resume this one.
|
||||
await expect(queue.schedule('same.com', async () => 'second')).resolves.toBe('second')
|
||||
})
|
||||
|
||||
test('stop() clears the pending timer so shutdown is not blocked', async () => {
|
||||
const queue = new HostQueue.Queue({ minSpacingMs: 10000, maxSpacingMs: 10000 })
|
||||
|
||||
await queue.schedule('same.com', async () => 'first')
|
||||
const queued = queue.schedule('same.com', async () => 'never')
|
||||
|
||||
queue.stop()
|
||||
|
||||
await expect(queued).rejects.toThrow('stopped')
|
||||
})
|
||||
})
|
||||
|
||||
describe('stats', () => {
|
||||
|
||||
test('reports pending, running and host counts', async () => {
|
||||
const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
|
||||
|
||||
const gate = deferred()
|
||||
const first = queue.schedule('a.com', () => gate.promise)
|
||||
const second = queue.schedule('b.com', async () => {})
|
||||
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
|
||||
const stats = queue.stats()
|
||||
expect(stats.running).toBe(1)
|
||||
expect(stats.pending).toBe(1)
|
||||
|
||||
gate.resolve()
|
||||
await Promise.all([first, second])
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user