Adding scrape files for testing
This commit is contained in:
Max committed
1 parent
7d255fe687
commit
8d60aae65c
40 files changed
+4423
No files matched your search
@@ -0,0 +1,195 @@
|
||||
//
|
||||
// Browser identities for outbound fetches.
|
||||
//
|
||||
// The old code sent a single hardcoded Googlebot User-Agent. That was removed
|
||||
// deliberately: Googlebot is verifiable by reverse DNS, so the claim is trivially
|
||||
// disproven, and sites that check it block the source IP outright. Some also
|
||||
// serve crawlers a stripped-down page.
|
||||
//
|
||||
// What replaces it is an honest, *coherent* browser identity. Coherence is the
|
||||
// point - a Chrome User-Agent shipping Firefox's Accept header, or sec-ch-ua on
|
||||
// a Safari request, is a stronger bot signal than any UA string. So each profile
|
||||
// carries the complete header set its family really sends, and nothing mixes.
|
||||
//
|
||||
// Profiles are assigned stably per host, not randomly per request: a site should
|
||||
// see one returning browser, not a rotating cast.
|
||||
//
|
||||
|
||||
const config = require('@config/scrape')
|
||||
|
||||
let BrowserProfiles = module.exports = {}
|
||||
|
||||
const CHROME_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'
|
||||
const FIREFOX_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
|
||||
const SAFARI_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
|
||||
|
||||
const PROFILES = BrowserProfiles.PROFILES = [
|
||||
{
|
||||
id: 'chrome-win',
|
||||
family: 'chrome',
|
||||
ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
||||
accept: CHROME_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.9',
|
||||
secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
|
||||
secChUaMobile: '?0',
|
||||
secChUaPlatform: '"Windows"',
|
||||
},
|
||||
{
|
||||
id: 'chrome-mac',
|
||||
family: 'chrome',
|
||||
ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
||||
accept: CHROME_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.9',
|
||||
secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
|
||||
secChUaMobile: '?0',
|
||||
secChUaPlatform: '"macOS"',
|
||||
},
|
||||
{
|
||||
id: 'firefox-win',
|
||||
family: 'firefox',
|
||||
ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
||||
accept: FIREFOX_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.5',
|
||||
},
|
||||
{
|
||||
id: 'firefox-linux',
|
||||
family: 'firefox',
|
||||
ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
||||
accept: FIREFOX_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.5',
|
||||
},
|
||||
{
|
||||
id: 'safari-mac',
|
||||
family: 'safari',
|
||||
ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
|
||||
accept: SAFARI_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.9',
|
||||
},
|
||||
]
|
||||
|
||||
// FNV-1a. Not cryptographic - we only need a stable, well-spread bucket.
|
||||
const fnv1a = (input) => {
|
||||
let hash = 0x811c9dc5
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
hash ^= input.charCodeAt(i)
|
||||
hash = Math.imul(hash, 0x01000193) >>> 0
|
||||
}
|
||||
return hash >>> 0
|
||||
}
|
||||
BrowserProfiles.__fnv1a = fnv1a
|
||||
|
||||
//
|
||||
// Same host always gets the same profile, across requests and across restarts
|
||||
// (the salt is a constant in config, not a per-boot random).
|
||||
//
|
||||
BrowserProfiles.profileForHost = (hostname) => {
|
||||
const key = require('@helpers/UrlSafety').registrableHost(hostname) + config.profileSalt
|
||||
return PROFILES[fnv1a(key) % PROFILES.length]
|
||||
}
|
||||
|
||||
BrowserProfiles.profileById = (id) => PROFILES.find(profile => profile.id === id) || PROFILES[0]
|
||||
|
||||
//
|
||||
// Sec-Fetch-Site is relative: "none" for a URL typed into the bar, "same-origin"
|
||||
// when the referrer shares the origin, "cross-site" otherwise. Getting this
|
||||
// wrong while sending a Referer is exactly the kind of incoherence that flags.
|
||||
//
|
||||
const fetchSite = (targetUrl, referer) => {
|
||||
if (!referer) { return 'none' }
|
||||
try {
|
||||
const target = new URL(targetUrl)
|
||||
const from = new URL(referer)
|
||||
if (target.origin === from.origin) { return 'same-origin' }
|
||||
if (UrlSafetyHost(target) === UrlSafetyHost(from)) { return 'same-site' }
|
||||
return 'cross-site'
|
||||
} catch (error) {
|
||||
return 'none'
|
||||
}
|
||||
}
|
||||
|
||||
const UrlSafetyHost = (url) => require('@helpers/UrlSafety').registrableHost(url.hostname)
|
||||
|
||||
const applyChromiumHints = (headers, profile) => {
|
||||
if (profile.family !== 'chrome') { return headers }
|
||||
headers['sec-ch-ua'] = profile.secChUa
|
||||
headers['sec-ch-ua-mobile'] = profile.secChUaMobile
|
||||
headers['sec-ch-ua-platform'] = profile.secChUaPlatform
|
||||
return headers
|
||||
}
|
||||
|
||||
//
|
||||
// Headers for a top-level document navigation.
|
||||
//
|
||||
BrowserProfiles.documentHeaders = (profile, { url, referer } = {}) => {
|
||||
const headers = {
|
||||
'User-Agent': profile.ua,
|
||||
'Accept': profile.accept,
|
||||
'Accept-Language': profile.acceptLanguage,
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
}
|
||||
|
||||
// Safari sends neither sec-ch-ua nor Sec-Fetch-*; Firefox sends Sec-Fetch-*
|
||||
// but no client hints; Chromium sends both.
|
||||
if (profile.family !== 'safari') {
|
||||
headers['Sec-Fetch-Dest'] = 'document'
|
||||
headers['Sec-Fetch-Mode'] = 'navigate'
|
||||
headers['Sec-Fetch-Site'] = fetchSite(url, referer)
|
||||
headers['Sec-Fetch-User'] = '?1'
|
||||
headers['Upgrade-Insecure-Requests'] = '1'
|
||||
}
|
||||
|
||||
applyChromiumHints(headers, profile)
|
||||
|
||||
if (referer) { headers['Referer'] = referer }
|
||||
|
||||
return headers
|
||||
}
|
||||
|
||||
//
|
||||
// Headers for a subresource image load, as issued from the page it was found on.
|
||||
//
|
||||
BrowserProfiles.imageHeaders = (profile, { url, pageUrl } = {}) => {
|
||||
const headers = {
|
||||
'User-Agent': profile.ua,
|
||||
'Accept': profile.family === 'chrome'
|
||||
? 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8'
|
||||
: 'image/avif,image/webp,*/*',
|
||||
'Accept-Language': profile.acceptLanguage,
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
}
|
||||
|
||||
if (profile.family !== 'safari') {
|
||||
headers['Sec-Fetch-Dest'] = 'image'
|
||||
headers['Sec-Fetch-Mode'] = 'no-cors'
|
||||
headers['Sec-Fetch-Site'] = fetchSite(url, pageUrl)
|
||||
}
|
||||
|
||||
applyChromiumHints(headers, profile)
|
||||
|
||||
if (pageUrl) { headers['Referer'] = pageUrl }
|
||||
|
||||
return headers
|
||||
}
|
||||
|
||||
//
|
||||
// Headers for robots.txt. Same identity as the page fetch, but it is not a
|
||||
// navigation - a browser would never send Sec-Fetch-Dest: document for this.
|
||||
//
|
||||
BrowserProfiles.robotsHeaders = (profile) => {
|
||||
const headers = {
|
||||
'User-Agent': profile.ua,
|
||||
'Accept': 'text/plain,*/*;q=0.8',
|
||||
'Accept-Language': profile.acceptLanguage,
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
}
|
||||
|
||||
if (profile.family !== 'safari') {
|
||||
headers['Sec-Fetch-Dest'] = 'empty'
|
||||
headers['Sec-Fetch-Mode'] = 'no-cors'
|
||||
headers['Sec-Fetch-Site'] = 'none'
|
||||
}
|
||||
|
||||
applyChromiumHints(headers, profile)
|
||||
|
||||
return headers
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
//
|
||||
// Per-host politeness scheduler. Knows nothing about scraping.
|
||||
//
|
||||
// This is the piece that stops the scraper looking like a crawler. The old code
|
||||
// did foundUrls.forEach(fetch) - a note with 40 links opened 40 simultaneous
|
||||
// connections, which is the most bot-like traffic shape available. Here, each
|
||||
// host gets one request at a time with jittered spacing between them, and a
|
||||
// global cap bounds total concurrency.
|
||||
//
|
||||
|
||||
const config = require('@config/scrape')
|
||||
|
||||
let HostQueue = module.exports = {}
|
||||
|
||||
const jitter = (min, max) => Math.round(min + Math.random() * Math.max(0, max - min))
|
||||
|
||||
class Queue {
|
||||
constructor(options = {}) {
|
||||
this.globalConcurrency = options.globalConcurrency != null
|
||||
? options.globalConcurrency : config.maxConcurrentFetches
|
||||
this.perHostConcurrency = options.perHostConcurrency != null
|
||||
? options.perHostConcurrency : config.perHostConcurrency
|
||||
this.minSpacingMs = options.minSpacingMs != null
|
||||
? options.minSpacingMs : config.perHostMinSpacingMs
|
||||
this.maxSpacingMs = options.maxSpacingMs != null
|
||||
? options.maxSpacingMs : config.perHostMaxSpacingMs
|
||||
this.now = options.now || (() => Date.now())
|
||||
// Injectable so tests with a controlled clock don't spend real
|
||||
// wall-time waiting out politeness spacing.
|
||||
this.setTimeoutFn = options.setTimeoutFn || ((fn, ms) => setTimeout(fn, ms))
|
||||
this.clearTimeoutFn = options.clearTimeoutFn || ((handle) => clearTimeout(handle))
|
||||
|
||||
this.pending = [] // FIFO
|
||||
this.hosts = new Map() // hostKey -> { running, nextAllowedAt }
|
||||
this.running = 0
|
||||
this.seq = 0
|
||||
this.timer = null
|
||||
this.stopped = false
|
||||
this.drainWaiters = []
|
||||
}
|
||||
|
||||
hostState(hostKey) {
|
||||
if (!this.hosts.has(hostKey)) {
|
||||
this.hosts.set(hostKey, { running: 0, nextAllowedAt: 0 })
|
||||
}
|
||||
return this.hosts.get(hostKey)
|
||||
}
|
||||
|
||||
//
|
||||
// Queue a task for a host. Resolves/rejects with the task's own outcome.
|
||||
// spacingMs overrides the jitter floor (robots.txt Crawl-delay).
|
||||
//
|
||||
schedule(hostKey, taskFn, options = {}) {
|
||||
if (this.stopped) {
|
||||
return Promise.reject(new Error('HostQueue stopped'))
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pending.push({
|
||||
hostKey: hostKey || '(none)',
|
||||
taskFn,
|
||||
resolve,
|
||||
reject,
|
||||
spacingMs: options.spacingMs != null ? options.spacingMs : null,
|
||||
seq: this.seq++,
|
||||
})
|
||||
this.pump()
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
// Run whatever is eligible. FIFO scan rather than per-host round-robin, so
|
||||
// no host can starve another and order stays predictable.
|
||||
//
|
||||
pump() {
|
||||
if (this.stopped) { return }
|
||||
|
||||
this.clearTimer()
|
||||
|
||||
while (this.running < this.globalConcurrency) {
|
||||
const at = this.now()
|
||||
let index = -1
|
||||
let earliest = Infinity
|
||||
|
||||
for (let i = 0; i < this.pending.length; i++) {
|
||||
const job = this.pending[i]
|
||||
const host = this.hostState(job.hostKey)
|
||||
|
||||
if (host.running >= this.perHostConcurrency) { continue }
|
||||
|
||||
if (host.nextAllowedAt > at) {
|
||||
earliest = Math.min(earliest, host.nextAllowedAt)
|
||||
continue
|
||||
}
|
||||
|
||||
index = i
|
||||
break
|
||||
}
|
||||
|
||||
if (index === -1) {
|
||||
// Nothing runnable now. Wake up when the earliest host is due.
|
||||
if (this.pending.length && earliest !== Infinity) {
|
||||
this.armTimer(earliest - at)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
this.run(this.pending.splice(index, 1)[0])
|
||||
}
|
||||
|
||||
this.checkDrained()
|
||||
}
|
||||
|
||||
run(job) {
|
||||
const host = this.hostState(job.hostKey)
|
||||
host.running++
|
||||
this.running++
|
||||
|
||||
const settle = () => {
|
||||
host.running--
|
||||
this.running--
|
||||
|
||||
// Space the NEXT request to this host from the end of this one.
|
||||
const spacing = job.spacingMs != null
|
||||
? job.spacingMs
|
||||
: jitter(this.minSpacingMs, this.maxSpacingMs)
|
||||
host.nextAllowedAt = this.now() + spacing
|
||||
|
||||
this.pump()
|
||||
}
|
||||
|
||||
// A throwing taskFn must not wedge the pump, so the call itself is
|
||||
// wrapped rather than only its promise.
|
||||
let result = null
|
||||
try {
|
||||
result = Promise.resolve(job.taskFn())
|
||||
} catch (error) {
|
||||
result = Promise.reject(error)
|
||||
}
|
||||
|
||||
result.then(
|
||||
value => { settle(); job.resolve(value) },
|
||||
error => { settle(); job.reject(error) }
|
||||
)
|
||||
}
|
||||
|
||||
armTimer(delay) {
|
||||
this.clearTimer()
|
||||
this.timer = this.setTimeoutFn(() => {
|
||||
this.timer = null
|
||||
this.pump()
|
||||
}, Math.max(1, delay))
|
||||
|
||||
//
|
||||
// Deliberately NOT unref'd.
|
||||
//
|
||||
// This timer is the only thing that will resume a host which is inside
|
||||
// its politeness gap. unref'ing it means that if nothing else holds the
|
||||
// event loop open, Node exits and every queued job is silently
|
||||
// abandoned with its promise never settling. Inside the server that is
|
||||
// masked by the listening socket, but it makes the queue unusable from
|
||||
// a standalone script and is a footgun either way.
|
||||
//
|
||||
// Shutdown is handled explicitly instead: stop() clears this timer and
|
||||
// rejects whatever is still queued, and it is wired to SIGINT/SIGTERM
|
||||
// in server/index.js.
|
||||
//
|
||||
}
|
||||
|
||||
clearTimer() {
|
||||
if (this.timer) {
|
||||
this.clearTimeoutFn(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
}
|
||||
|
||||
size() { return this.pending.length }
|
||||
inflight() { return this.running }
|
||||
|
||||
stats() {
|
||||
return {
|
||||
pending: this.pending.length,
|
||||
running: this.running,
|
||||
hosts: this.hosts.size,
|
||||
}
|
||||
}
|
||||
|
||||
checkDrained() {
|
||||
if (this.pending.length || this.running) { return }
|
||||
|
||||
const waiters = this.drainWaiters
|
||||
this.drainWaiters = []
|
||||
waiters.forEach(resolve => resolve())
|
||||
}
|
||||
|
||||
// Resolves once the queue is empty and nothing is in flight.
|
||||
drain() {
|
||||
if (!this.pending.length && !this.running) { return Promise.resolve() }
|
||||
return new Promise(resolve => this.drainWaiters.push(resolve))
|
||||
}
|
||||
|
||||
//
|
||||
// Reject everything still queued. In-flight tasks are not cancelled here -
|
||||
// their own AbortControllers handle that.
|
||||
//
|
||||
stop() {
|
||||
this.stopped = true
|
||||
this.clearTimer()
|
||||
|
||||
const pending = this.pending
|
||||
this.pending = []
|
||||
pending.forEach(job => job.reject(new Error('HostQueue stopped')))
|
||||
|
||||
this.checkDrained()
|
||||
}
|
||||
}
|
||||
|
||||
HostQueue.Queue = Queue
|
||||
|
||||
// Shared instance used by the scrape worker. Tests build their own.
|
||||
HostQueue.shared = new Queue()
|
||||
@@ -0,0 +1,429 @@
|
||||
//
|
||||
// The only module in the server that opens an outbound connection.
|
||||
//
|
||||
// Everything that made the old scraper unreliable is handled here, in one place:
|
||||
// a single AbortController-based timeout (the old code raced request's socket
|
||||
// timeout against a manual setTimeout that could fire against a null reference
|
||||
// and crash the process), manual redirect following so the SSRF gate runs on
|
||||
// every hop, hard size caps enforced while streaming, a Content-Type gate before
|
||||
// any body is read, real charset decoding, and retries that only fire for
|
||||
// transient failures.
|
||||
//
|
||||
|
||||
const config = require('@config/scrape')
|
||||
const UrlSafety = require('@helpers/UrlSafety')
|
||||
const BrowserProfiles = require('@helpers/BrowserProfiles')
|
||||
|
||||
let HttpFetch = module.exports = {}
|
||||
|
||||
const HTML_TYPES = ['text/html', 'application/xhtml+xml', 'text/plain', 'application/xml', 'text/xml']
|
||||
|
||||
class FetchError extends Error {
|
||||
constructor(kind, message, extra = {}) {
|
||||
super(message)
|
||||
this.name = 'FetchError'
|
||||
this.kind = kind // timeout|http|blocked|too_large|bad_content_type|redirect_loop|too_many_redirects|network|aborted
|
||||
this.status = extra.status || null
|
||||
this.retryAfterMs = extra.retryAfterMs || null
|
||||
this.url = extra.url || null
|
||||
this.reason = extra.reason || null
|
||||
}
|
||||
}
|
||||
HttpFetch.FetchError = FetchError
|
||||
|
||||
// Injectable so tests don't spend real wall-time in backoff. Far simpler than
|
||||
// interleaving fake timers with promises.
|
||||
let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
|
||||
HttpFetch.__setSleep = (fn) => { sleep = fn }
|
||||
HttpFetch.__resetSleep = () => { sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) }
|
||||
|
||||
const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504, 507, 509, 521, 522, 523, 524])
|
||||
|
||||
const isRetryable = (error) => {
|
||||
if (!(error instanceof FetchError)) { return false }
|
||||
if (error.kind === 'network' || error.kind === 'timeout') { return true }
|
||||
if (error.kind === 'http') { return RETRYABLE_STATUS.has(error.status) }
|
||||
return false
|
||||
}
|
||||
|
||||
// Retry-After is either delta-seconds or an HTTP-date.
|
||||
const parseRetryAfter = (value) => {
|
||||
if (!value) { return null }
|
||||
|
||||
const seconds = Number(value)
|
||||
if (Number.isFinite(seconds)) { return Math.max(0, seconds * 1000) }
|
||||
|
||||
const when = Date.parse(value)
|
||||
if (Number.isFinite(when)) { return Math.max(0, when - Date.now()) }
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
const backoffFor = (attempt) => {
|
||||
const base = Math.min(config.backoffMaxMs, config.backoffBaseMs * Math.pow(2, attempt))
|
||||
return Math.round(base * (0.5 + Math.random())) // +/-50% jitter
|
||||
}
|
||||
|
||||
const contentTypeOf = (response) => String(response.headers.get('content-type') || '').toLowerCase()
|
||||
const mimeOf = (contentType) => contentType.split(';')[0].trim()
|
||||
|
||||
const charsetFromContentType = (contentType) => {
|
||||
const match = contentType.match(/charset=\s*"?([^";\s]+)"?/i)
|
||||
return match ? match[1].toLowerCase() : null
|
||||
}
|
||||
|
||||
// Last resort: sniff a <meta charset> out of the head, read as latin1 so every
|
||||
// byte maps to a character and the regex can't miss.
|
||||
const charsetFromMeta = (buffer) => {
|
||||
const head = buffer.subarray(0, 2048).toString('latin1')
|
||||
|
||||
const metaCharset = head.match(/<meta[^>]+charset\s*=\s*["']?\s*([a-z0-9_\-:]+)/i)
|
||||
if (metaCharset) { return metaCharset[1].toLowerCase() }
|
||||
|
||||
const httpEquiv = head.match(/<meta[^>]+content\s*=\s*["'][^"']*charset=\s*([a-z0-9_\-:]+)/i)
|
||||
if (httpEquiv) { return httpEquiv[1].toLowerCase() }
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
const charsetFromBom = (buffer) => {
|
||||
if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) { return 'utf-8' }
|
||||
if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { return 'utf-16le' }
|
||||
if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) { return 'utf-16be' }
|
||||
return null
|
||||
}
|
||||
|
||||
const decodeBody = (buffer, declaredCharset) => {
|
||||
const label = declaredCharset || charsetFromBom(buffer) || charsetFromMeta(buffer) || 'utf-8'
|
||||
|
||||
try {
|
||||
return { text: new TextDecoder(label, { fatal: false }).decode(buffer), charset: label }
|
||||
} catch (error) {
|
||||
// Unknown/bogus label - utf-8 with replacement chars beats throwing.
|
||||
return { text: new TextDecoder('utf-8', { fatal: false }).decode(buffer), charset: 'utf-8' }
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Read the body with a running byte count, aborting mid-stream once the cap is
|
||||
// passed. Content-Length is checked by the caller too, but it is optional and
|
||||
// can lie, so this is the cap that actually holds.
|
||||
//
|
||||
const readCapped = async (response, maxBytes, controller, url) => {
|
||||
if (!response.body) { return Buffer.alloc(0) }
|
||||
|
||||
const chunks = []
|
||||
let total = 0
|
||||
|
||||
for await (const chunk of response.body) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
||||
total += buffer.length
|
||||
|
||||
if (total > maxBytes) {
|
||||
try { controller.abort() } catch (error) { /* already done */ }
|
||||
throw new FetchError('too_large', `Body exceeded ${maxBytes} bytes`, { url })
|
||||
}
|
||||
|
||||
chunks.push(buffer)
|
||||
}
|
||||
|
||||
return Buffer.concat(chunks, total)
|
||||
}
|
||||
|
||||
//
|
||||
// One HTTP request, following redirects by hand.
|
||||
//
|
||||
// Manual redirects are the whole point: fetch's own 'follow' would hop to
|
||||
// 127.0.0.1 without ever re-consulting the safety gate.
|
||||
//
|
||||
const requestOnce = async (startUrl, options) => {
|
||||
const {
|
||||
maxBytes,
|
||||
acceptMimes,
|
||||
headersFor,
|
||||
conditional,
|
||||
timeoutMs,
|
||||
deadline,
|
||||
referer,
|
||||
} = options
|
||||
|
||||
let currentUrl = typeof startUrl === 'string' ? startUrl : startUrl.href
|
||||
const visited = new Set()
|
||||
|
||||
for (let hop = 0; hop <= config.maxRedirects; hop++) {
|
||||
if (visited.has(currentUrl)) {
|
||||
throw new FetchError('redirect_loop', `Redirect loop at ${currentUrl}`, { url: currentUrl })
|
||||
}
|
||||
visited.add(currentUrl)
|
||||
|
||||
// Re-validate on every hop. This is the check the old code never had.
|
||||
try {
|
||||
await UrlSafety.assertFetchable(currentUrl)
|
||||
} catch (error) {
|
||||
// A DNS failure is transient - the host may simply be down right
|
||||
// now - so it must not be classified alongside "this address is
|
||||
// private", which is a permanent refusal. Collapsing the two meant
|
||||
// a briefly-unresolvable domain got parked for the full cache TTL
|
||||
// and had its attachment row deleted.
|
||||
const kind = error.reason === 'dns_failed' ? 'network' : 'blocked'
|
||||
throw new FetchError(kind, error.message, { url: currentUrl, reason: error.reason })
|
||||
}
|
||||
|
||||
const remaining = deadline - Date.now()
|
||||
if (remaining <= 0) {
|
||||
throw new FetchError('timeout', 'Total budget exhausted', { url: currentUrl })
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, remaining))
|
||||
|
||||
let response = null
|
||||
try {
|
||||
const headers = headersFor(currentUrl, referer)
|
||||
|
||||
if (conditional && conditional.etag) { headers['If-None-Match'] = conditional.etag }
|
||||
if (conditional && conditional.lastModified) { headers['If-Modified-Since'] = conditional.lastModified }
|
||||
|
||||
response = await fetch(currentUrl, {
|
||||
method: 'GET',
|
||||
redirect: 'manual',
|
||||
signal: controller.signal,
|
||||
headers,
|
||||
})
|
||||
} catch (error) {
|
||||
clearTimeout(timer)
|
||||
|
||||
if (controller.signal.aborted) {
|
||||
throw new FetchError('timeout', `Timed out after ${timeoutMs}ms`, { url: currentUrl })
|
||||
}
|
||||
throw new FetchError('network', error.message || 'Network failure', { url: currentUrl })
|
||||
}
|
||||
|
||||
// Redirect?
|
||||
if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
|
||||
clearTimeout(timer)
|
||||
try { if (response.body) { await response.body.cancel() } } catch (error) { /* nothing to drain */ }
|
||||
|
||||
const location = response.headers.get('location')
|
||||
let next = null
|
||||
try {
|
||||
next = new URL(location, currentUrl)
|
||||
} catch (error) {
|
||||
throw new FetchError('network', `Unparseable redirect target ${location}`, { url: currentUrl })
|
||||
}
|
||||
|
||||
// Never follow https -> http. A downgrade mid-chain is either an
|
||||
// attack or a misconfiguration, and is not worth the preview.
|
||||
if (new URL(currentUrl).protocol === 'https:' && next.protocol === 'http:') {
|
||||
throw new FetchError('blocked', `Refusing https->http downgrade to ${next.href}`, { url: currentUrl })
|
||||
}
|
||||
|
||||
currentUrl = next.href
|
||||
continue
|
||||
}
|
||||
|
||||
clearTimeout(timer)
|
||||
|
||||
if (response.status === 304) {
|
||||
try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
|
||||
return { notModified: true, status: 304, finalUrl: currentUrl, response }
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
|
||||
throw new FetchError('http', `HTTP ${response.status} for ${currentUrl}`, {
|
||||
status: response.status,
|
||||
retryAfterMs: parseRetryAfter(response.headers.get('retry-after')),
|
||||
url: currentUrl,
|
||||
})
|
||||
}
|
||||
|
||||
const contentType = contentTypeOf(response)
|
||||
const mime = mimeOf(contentType)
|
||||
|
||||
// Gate on type BEFORE reading a byte. This is what stops a 100MB binary
|
||||
// being handed to cheerio.
|
||||
if (acceptMimes && !acceptMimes(mime)) {
|
||||
try { controller.abort() } catch (error) { /* ignore */ }
|
||||
throw new FetchError('bad_content_type', `Refusing content-type ${mime || '(none)'}`, { url: currentUrl })
|
||||
}
|
||||
|
||||
const declaredLength = Number(response.headers.get('content-length'))
|
||||
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
||||
try { controller.abort() } catch (error) { /* ignore */ }
|
||||
throw new FetchError('too_large', `Declared length ${declaredLength} over cap`, { url: currentUrl })
|
||||
}
|
||||
|
||||
const buffer = await readCapped(response, maxBytes, controller, currentUrl)
|
||||
|
||||
return {
|
||||
notModified: false,
|
||||
status: response.status,
|
||||
finalUrl: currentUrl,
|
||||
contentType,
|
||||
mime,
|
||||
buffer,
|
||||
response,
|
||||
}
|
||||
}
|
||||
|
||||
throw new FetchError('too_many_redirects', `More than ${config.maxRedirects} redirects`, { url: currentUrl })
|
||||
}
|
||||
|
||||
//
|
||||
// Retry wrapper. Only transient classes are retried - a 404 is an answer.
|
||||
//
|
||||
const withRetries = async (attemptFn, { retries, deadline }) => {
|
||||
let lastError = null
|
||||
|
||||
for (let attempt = 0; attempt <= retries; attempt++) {
|
||||
try {
|
||||
return await attemptFn()
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
|
||||
if (!isRetryable(error) || attempt === retries) { throw error }
|
||||
|
||||
// A server asking us to come back in an hour is not something to
|
||||
// sit on - surface it so the cache can park the URL instead.
|
||||
if (error.retryAfterMs && error.retryAfterMs > config.maxRetryAfterMs) { throw error }
|
||||
|
||||
const wait = error.retryAfterMs != null
|
||||
? Math.min(error.retryAfterMs, config.maxRetryAfterMs)
|
||||
: backoffFor(attempt)
|
||||
|
||||
if (Date.now() + wait >= deadline) { throw error }
|
||||
|
||||
await sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError
|
||||
}
|
||||
|
||||
//
|
||||
// Fetch an HTML document.
|
||||
//
|
||||
HttpFetch.fetchDocument = async (url, options = {}) => {
|
||||
const timeoutMs = options.timeoutMs || config.pageTimeoutMs
|
||||
const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
|
||||
const maxBytes = options.maxBytes || config.maxHtmlBytes
|
||||
const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
|
||||
|
||||
const result = await withRetries(() => requestOnce(url, {
|
||||
maxBytes,
|
||||
acceptMimes: (mime) => HTML_TYPES.includes(mime),
|
||||
headersFor: (target, referer) => BrowserProfiles.documentHeaders(profile, {
|
||||
url: target,
|
||||
// A browser arriving at a page has *some* referer; its own origin is
|
||||
// the least surprising one and pairs with Sec-Fetch-Site: same-origin.
|
||||
referer: referer || originOf(target),
|
||||
}),
|
||||
conditional: options.conditional,
|
||||
timeoutMs,
|
||||
deadline,
|
||||
referer: options.referer,
|
||||
}), { retries: options.retries != null ? options.retries : config.retries, deadline })
|
||||
|
||||
if (result.notModified) {
|
||||
return {
|
||||
notModified: true,
|
||||
status: 304,
|
||||
finalUrl: result.finalUrl,
|
||||
html: null,
|
||||
profileId: profile.id,
|
||||
}
|
||||
}
|
||||
|
||||
const declaredCharset = charsetFromContentType(result.contentType)
|
||||
const { text, charset } = decodeBody(result.buffer, declaredCharset)
|
||||
|
||||
return {
|
||||
notModified: false,
|
||||
status: result.status,
|
||||
finalUrl: result.finalUrl,
|
||||
html: text,
|
||||
charset,
|
||||
contentType: result.contentType,
|
||||
bytes: result.buffer.length,
|
||||
etag: result.response.headers.get('etag') || null,
|
||||
lastModified: result.response.headers.get('last-modified') || null,
|
||||
profileId: profile.id,
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Fetch a binary (thumbnail image).
|
||||
//
|
||||
HttpFetch.fetchBinary = async (url, options = {}) => {
|
||||
const timeoutMs = options.timeoutMs || config.imageTimeoutMs
|
||||
const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
|
||||
const maxBytes = options.maxBytes || config.maxImageBytes
|
||||
const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
|
||||
|
||||
const acceptMimes = (mime) => {
|
||||
if (!mime.startsWith('image/')) { return false }
|
||||
if (mime === 'image/svg+xml' && !config.allowSvgThumbnails) { return false }
|
||||
return true
|
||||
}
|
||||
|
||||
const result = await withRetries(() => requestOnce(url, {
|
||||
maxBytes,
|
||||
acceptMimes,
|
||||
headersFor: (target) => BrowserProfiles.imageHeaders(profile, {
|
||||
url: target,
|
||||
pageUrl: options.pageUrl,
|
||||
}),
|
||||
timeoutMs,
|
||||
deadline,
|
||||
}), { retries: options.retries != null ? options.retries : config.retries, deadline })
|
||||
|
||||
return {
|
||||
status: result.status,
|
||||
finalUrl: result.finalUrl,
|
||||
contentType: result.contentType,
|
||||
mime: result.mime,
|
||||
buffer: result.buffer,
|
||||
profileId: profile.id,
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Fetch robots.txt. Short leash, no retries - fail-open is handled by the caller.
|
||||
//
|
||||
HttpFetch.fetchText = async (url, options = {}) => {
|
||||
const timeoutMs = options.timeoutMs || 5000
|
||||
const deadline = Date.now() + (options.budgetMs || timeoutMs + 1000)
|
||||
const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
|
||||
|
||||
const result = await requestOnce(url, {
|
||||
maxBytes: options.maxBytes || config.maxRobotsBytes,
|
||||
acceptMimes: null, // robots.txt is served as every type under the sun
|
||||
headersFor: () => BrowserProfiles.robotsHeaders(profile),
|
||||
timeoutMs,
|
||||
deadline,
|
||||
})
|
||||
|
||||
if (result.notModified) { return { status: 304, text: null, finalUrl: result.finalUrl } }
|
||||
|
||||
const { text } = decodeBody(result.buffer, charsetFromContentType(result.contentType))
|
||||
return { status: result.status, text, finalUrl: result.finalUrl }
|
||||
}
|
||||
|
||||
const hostnameOf = (url) => {
|
||||
try { return new URL(typeof url === 'string' ? url : url.href).hostname }
|
||||
catch (error) { return '' }
|
||||
}
|
||||
|
||||
const originOf = (url) => {
|
||||
try { return new URL(url).origin + '/' }
|
||||
catch (error) { return undefined }
|
||||
}
|
||||
|
||||
//
|
||||
// Deliberately NOT attempted here, so nobody wonders later:
|
||||
// - Header ordering: fetch normalises it; controlling order needs raw sockets.
|
||||
// - HTTP/2: undici's fetch is h1 unless an Agent with allowH2 is supplied.
|
||||
// - TLS/JA3 fingerprinting and proxy rotation: ruled out, we are being polite
|
||||
// rather than evasive.
|
||||
//
|
||||
@@ -0,0 +1,43 @@
|
||||
//
|
||||
// Socket.IO emit wrapper.
|
||||
//
|
||||
// Replaces the `if(typeof SocketIo != 'undefined')` guards that were scattered
|
||||
// through the models. Those guards did not just avoid a crash - they silently
|
||||
// disabled ALL scraping under Jest, which is why none of this code was ever
|
||||
// testable. Here the no-socket case is a no-op rather than a skipped branch, so
|
||||
// the surrounding logic runs identically in tests and in production.
|
||||
//
|
||||
|
||||
let Realtime = module.exports = {}
|
||||
|
||||
let warned = false
|
||||
|
||||
const server = () => (typeof global.SocketIo !== 'undefined' && global.SocketIo) ? global.SocketIo : null
|
||||
|
||||
Realtime.isEnabled = () => server() !== null
|
||||
|
||||
const emit = (room, event, payload) => {
|
||||
const io = server()
|
||||
|
||||
if (!io) {
|
||||
if (!warned) {
|
||||
warned = true
|
||||
console.log('[realtime] no socket server bound; emits are no-ops')
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
if (payload === undefined) { io.to(room).emit(event) }
|
||||
else { io.to(room).emit(event, payload) }
|
||||
return true
|
||||
} catch (error) {
|
||||
console.log('[realtime] emit failed', event, error.message)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
Realtime.emitToUser = (userId, event, payload) => emit(`user:${userId}`, event, payload)
|
||||
Realtime.emitToNote = (noteRawTextId, event, payload) => emit(`note:${noteRawTextId}`, event, payload)
|
||||
|
||||
Realtime.__resetWarning = () => { warned = false }
|
||||
@@ -0,0 +1,216 @@
|
||||
//
|
||||
// robots.txt fetching, parsing and caching.
|
||||
//
|
||||
// Posture: polite, not evasive. An explicit Disallow is respected (fail-closed);
|
||||
// a robots.txt we cannot retrieve is treated as permissive (fail-open), because
|
||||
// a 404 or a flaky server is not a refusal and these are single-page fetches of
|
||||
// URLs a user explicitly saved.
|
||||
//
|
||||
|
||||
const config = require('@config/scrape')
|
||||
const HttpFetch = require('@helpers/HttpFetch')
|
||||
|
||||
let Robots = module.exports = {}
|
||||
|
||||
// origin -> { rules, expiresAt }
|
||||
const cache = new Map()
|
||||
|
||||
Robots.__resetCache = () => cache.clear()
|
||||
Robots.__cacheSize = () => cache.size
|
||||
|
||||
const ALLOW_ALL = { groups: [], allowAll: true }
|
||||
|
||||
//
|
||||
// Parse robots.txt into user-agent groups.
|
||||
//
|
||||
// Consecutive User-agent lines share one rule block, which is the part naive
|
||||
// parsers get wrong.
|
||||
//
|
||||
Robots.parse = (text) => {
|
||||
if (!text || typeof text !== 'string') { return ALLOW_ALL }
|
||||
|
||||
const groups = []
|
||||
let current = null
|
||||
let expectingAgents = false
|
||||
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = rawLine.replace(/#.*$/, '').trim()
|
||||
if (!line) { continue }
|
||||
|
||||
const separator = line.indexOf(':')
|
||||
if (separator === -1) { continue }
|
||||
|
||||
const field = line.slice(0, separator).trim().toLowerCase()
|
||||
const value = line.slice(separator + 1).trim()
|
||||
|
||||
if (field === 'user-agent') {
|
||||
// A User-agent line directly after another starts a shared group.
|
||||
if (!current || !expectingAgents) {
|
||||
current = { agents: [], rules: [], crawlDelayMs: null }
|
||||
groups.push(current)
|
||||
}
|
||||
current.agents.push(value.toLowerCase())
|
||||
expectingAgents = true
|
||||
continue
|
||||
}
|
||||
|
||||
if (!current) { continue } // directive before any user-agent
|
||||
expectingAgents = false
|
||||
|
||||
if (field === 'disallow' || field === 'allow') {
|
||||
// "Disallow:" with an empty value means allow everything.
|
||||
if (field === 'disallow' && value === '') { continue }
|
||||
current.rules.push({ allow: field === 'allow', path: value })
|
||||
continue
|
||||
}
|
||||
|
||||
if (field === 'crawl-delay') {
|
||||
const seconds = Number(value)
|
||||
if (Number.isFinite(seconds) && seconds >= 0) {
|
||||
current.crawlDelayMs = Math.round(seconds * 1000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { groups, allowAll: groups.length === 0 }
|
||||
}
|
||||
|
||||
//
|
||||
// Pick the group whose agent token best matches us. A named match always beats
|
||||
// the "*" wildcard, regardless of file order.
|
||||
//
|
||||
const groupFor = (rules, userAgentToken) => {
|
||||
if (!rules || rules.allowAll) { return null }
|
||||
|
||||
const token = userAgentToken.toLowerCase()
|
||||
let best = null
|
||||
let bestScore = -1
|
||||
|
||||
for (const group of rules.groups) {
|
||||
for (const agent of group.agents) {
|
||||
let score = -1
|
||||
if (agent === '*') { score = 0 }
|
||||
else if (token.includes(agent)) { score = agent.length }
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score
|
||||
best = group
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
//
|
||||
// Match a robots path pattern, supporting the * and $ wildcards.
|
||||
//
|
||||
const matchesPattern = (pattern, path) => {
|
||||
if (pattern === '') { return false }
|
||||
if (!pattern.includes('*') && !pattern.includes('$')) {
|
||||
return path.startsWith(pattern)
|
||||
}
|
||||
|
||||
const anchored = pattern.endsWith('$')
|
||||
const body = anchored ? pattern.slice(0, -1) : pattern
|
||||
|
||||
const expression = body
|
||||
.split('*')
|
||||
.map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
|
||||
.join('.*')
|
||||
|
||||
return new RegExp('^' + expression + (anchored ? '$' : '')).test(path)
|
||||
}
|
||||
|
||||
//
|
||||
// Longest matching rule wins; Allow beats Disallow at equal length.
|
||||
//
|
||||
Robots.isPathAllowed = (rules, path, userAgentToken) => {
|
||||
const group = groupFor(rules, userAgentToken)
|
||||
if (!group) { return true }
|
||||
|
||||
let decision = true
|
||||
let decisionLength = -1
|
||||
|
||||
for (const rule of group.rules) {
|
||||
if (!matchesPattern(rule.path, path)) { continue }
|
||||
|
||||
const length = rule.path.length
|
||||
if (length > decisionLength || (length === decisionLength && rule.allow)) {
|
||||
decision = rule.allow
|
||||
decisionLength = length
|
||||
}
|
||||
}
|
||||
|
||||
return decision
|
||||
}
|
||||
|
||||
const evict = () => {
|
||||
if (cache.size <= config.robotsMaxOrigins) { return }
|
||||
// Map preserves insertion order, so the oldest key is first.
|
||||
const oldest = cache.keys().next().value
|
||||
cache.delete(oldest)
|
||||
}
|
||||
|
||||
const getRules = async (origin) => {
|
||||
const cached = cache.get(origin)
|
||||
if (cached && cached.expiresAt > Date.now()) { return cached.rules }
|
||||
|
||||
let rules = ALLOW_ALL
|
||||
let ttl = config.robotsTtlMs
|
||||
|
||||
try {
|
||||
const result = await HttpFetch.fetchText(`${origin}/robots.txt`, {
|
||||
maxBytes: config.maxRobotsBytes,
|
||||
})
|
||||
rules = Robots.parse(result.text)
|
||||
} catch (error) {
|
||||
// No robots.txt, or we could not reach it. Permissive, but cached for a
|
||||
// short while only so a transient outage does not grant a day's pass.
|
||||
rules = ALLOW_ALL
|
||||
ttl = (error && error.kind === 'http' && error.status === 404)
|
||||
? config.robotsNegativeTtlMs
|
||||
: config.robotsErrorTtlMs
|
||||
}
|
||||
|
||||
cache.set(origin, { rules, expiresAt: Date.now() + ttl })
|
||||
evict()
|
||||
|
||||
return rules
|
||||
}
|
||||
Robots.__getRules = getRules
|
||||
|
||||
const originAndPath = (url) => {
|
||||
const parsed = new URL(typeof url === 'string' ? url : url.href)
|
||||
return { origin: parsed.origin, path: parsed.pathname + parsed.search }
|
||||
}
|
||||
|
||||
//
|
||||
// Is this URL allowed by its host's robots.txt?
|
||||
//
|
||||
Robots.isAllowed = async (url, userAgentToken) => {
|
||||
let parts = null
|
||||
try { parts = originAndPath(url) } catch (error) { return false }
|
||||
|
||||
const rules = await getRules(parts.origin)
|
||||
return Robots.isPathAllowed(rules, parts.path, userAgentToken || config.robotsUaToken)
|
||||
}
|
||||
|
||||
//
|
||||
// Crawl-delay for this host, or null. Capped: a site asking for an hour between
|
||||
// requests gets its job failed rather than parked in the queue forever.
|
||||
//
|
||||
Robots.crawlDelayMs = async (url, userAgentToken) => {
|
||||
let parts = null
|
||||
try { parts = originAndPath(url) } catch (error) { return null }
|
||||
|
||||
const rules = await getRules(parts.origin)
|
||||
const group = groupFor(rules, (userAgentToken || config.robotsUaToken).toLowerCase())
|
||||
|
||||
if (!group || group.crawlDelayMs == null) { return null }
|
||||
return group.crawlDelayMs
|
||||
}
|
||||
|
||||
Robots.exceedsCrawlDelayCap = (delayMs) => {
|
||||
return delayMs != null && delayMs > config.maxCrawlDelayMs
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//
|
||||
// Idempotent schema convergence for the scraper tables.
|
||||
//
|
||||
// There is no migration framework in this project and no schema file in git,
|
||||
// so this runs at boot, checks information_schema for each object, and applies
|
||||
// only what is missing. Dev and prod self-heal, and deploy.sh needs no change.
|
||||
//
|
||||
// Portability matters here: dev is MariaDB 11.4 and prod is MySQL 8.
|
||||
// ADD COLUMN IF NOT EXISTS is MariaDB-only and is deliberately not used -
|
||||
// every ALTER is guarded by an explicit existence check instead.
|
||||
//
|
||||
|
||||
const db = require('@config/database')
|
||||
|
||||
let Schema = module.exports = {}
|
||||
|
||||
const SCRAPE_CACHE_TABLE = `
|
||||
CREATE TABLE IF NOT EXISTS scrape_cache (
|
||||
id int(10) unsigned NOT NULL AUTO_INCREMENT,
|
||||
url_hash char(64) NOT NULL,
|
||||
url varchar(1000) NOT NULL DEFAULT '',
|
||||
canonical_url varchar(1000) DEFAULT NULL,
|
||||
status smallint(6) NOT NULL DEFAULT 0,
|
||||
title varchar(512) DEFAULT NULL,
|
||||
site_name varchar(255) DEFAULT NULL,
|
||||
description mediumtext DEFAULT NULL,
|
||||
keywords varchar(512) DEFAULT NULL,
|
||||
search_text mediumtext DEFAULT NULL,
|
||||
image_url varchar(1000) DEFAULT NULL,
|
||||
file_location varchar(500) DEFAULT NULL,
|
||||
etag varchar(255) DEFAULT NULL,
|
||||
last_modified varchar(128) DEFAULT NULL,
|
||||
content_type varchar(128) DEFAULT NULL,
|
||||
fetch_state varchar(20) NOT NULL DEFAULT 'pending',
|
||||
failure_count int(11) NOT NULL DEFAULT 0,
|
||||
last_error varchar(255) DEFAULT NULL,
|
||||
fetched_at int(11) DEFAULT NULL,
|
||||
expires_at int(11) DEFAULT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY scrape_cache_url_hash (url_hash),
|
||||
KEY scrape_cache_expires_at (expires_at),
|
||||
KEY scrape_cache_fetch_state (fetch_state)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
`
|
||||
|
||||
// DEFAULT 2 ("done") keeps existing rows out of the boot recovery sweep.
|
||||
const ATTACHMENT_COLUMNS = [
|
||||
{ name: 'scrape_state', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_state tinyint(4) NOT NULL DEFAULT 2' },
|
||||
{ name: 'scrape_attempts', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_attempts tinyint(4) NOT NULL DEFAULT 0' },
|
||||
{ name: 'enqueued_at', ddl: 'ALTER TABLE attachment ADD COLUMN enqueued_at int(11) DEFAULT NULL' },
|
||||
]
|
||||
|
||||
const ATTACHMENT_INDEXES = [
|
||||
{ name: 'attachment_scrape_state', ddl: 'ALTER TABLE attachment ADD KEY attachment_scrape_state (scrape_state, enqueued_at)' },
|
||||
]
|
||||
|
||||
const columnExists = async (table, column) => {
|
||||
const [rows] = await db.promise().query(
|
||||
`SELECT 1 FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`,
|
||||
[table, column]
|
||||
)
|
||||
return rows.length > 0
|
||||
}
|
||||
|
||||
const indexExists = async (table, index) => {
|
||||
const [rows] = await db.promise().query(
|
||||
`SELECT 1 FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1`,
|
||||
[table, index]
|
||||
)
|
||||
return rows.length > 0
|
||||
}
|
||||
Schema.__columnExists = columnExists
|
||||
Schema.__indexExists = indexExists
|
||||
|
||||
//
|
||||
// Safe to call repeatedly. Returns the list of statements it actually applied,
|
||||
// so boot logging says something useful on the one run that matters.
|
||||
//
|
||||
Schema.ensureScrapeSchema = async () => {
|
||||
const applied = []
|
||||
|
||||
await db.promise().query(SCRAPE_CACHE_TABLE)
|
||||
|
||||
for (const column of ATTACHMENT_COLUMNS) {
|
||||
if (await columnExists('attachment', column.name)) { continue }
|
||||
await db.promise().query(column.ddl)
|
||||
applied.push(`attachment.${column.name}`)
|
||||
}
|
||||
|
||||
// Index last: it references the columns above.
|
||||
for (const index of ATTACHMENT_INDEXES) {
|
||||
if (await indexExists('attachment', index.name)) { continue }
|
||||
await db.promise().query(index.ddl)
|
||||
applied.push(`attachment KEY ${index.name}`)
|
||||
}
|
||||
|
||||
if (applied.length) {
|
||||
console.log('[schema] applied scrape schema changes:', applied.join(', '))
|
||||
}
|
||||
|
||||
return applied
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
//
|
||||
// URL normalisation and the SSRF gate.
|
||||
//
|
||||
// Scraped URLs come from untrusted note text and from the /pushmebaby query
|
||||
// param, so every URL is validated before a socket is opened - and again after
|
||||
// every redirect hop, which is the hop that actually matters (a public URL that
|
||||
// 302s to 127.0.0.1 is the classic bypass).
|
||||
//
|
||||
// Pure except for assertFetchable, which resolves DNS.
|
||||
//
|
||||
|
||||
const crypto = require('crypto')
|
||||
const dns = require('dns').promises
|
||||
const net = require('net')
|
||||
const config = require('@config/scrape')
|
||||
|
||||
let UrlSafety = module.exports = {}
|
||||
|
||||
const FETCHABLE_SCHEMES = new Set(['http:', 'https:'])
|
||||
|
||||
// Hostname suffixes that never belong to a public site.
|
||||
const BLOCKED_SUFFIXES = ['.localhost', '.local', '.internal', '.home.arpa', '.lan']
|
||||
|
||||
// Query params that change nothing about the page but would fragment the cache.
|
||||
const TRACKING_PARAMS = [
|
||||
/^utm_/i, /^fbclid$/i, /^gclid$/i, /^dclid$/i, /^msclkid$/i,
|
||||
/^mc_(e|c)id$/i, /^igshid$/i, /^ref_src$/i, /^_hs(enc|mi)$/i, /^vero_/i,
|
||||
]
|
||||
|
||||
class SafetyError extends Error {
|
||||
constructor(reason, message) {
|
||||
super(message || reason)
|
||||
this.name = 'SafetyError'
|
||||
this.reason = reason
|
||||
}
|
||||
}
|
||||
UrlSafety.SafetyError = SafetyError
|
||||
|
||||
//
|
||||
// Parse to a URL object, or null. Does NOT mutate anything that would change
|
||||
// which resource is addressed - callers may store the result.
|
||||
//
|
||||
UrlSafety.parse = (raw, base) => {
|
||||
if (!raw || typeof raw !== 'string') { return null }
|
||||
|
||||
let url = null
|
||||
try {
|
||||
url = new URL(raw.trim(), base || undefined)
|
||||
} catch (error) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (!FETCHABLE_SCHEMES.has(url.protocol)) { return null }
|
||||
if (url.href.length > config.maxUrlLength) { return null }
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
//
|
||||
// Canonical form used ONLY for cache keys. Never store this - it deliberately
|
||||
// discards tracking params and case, which would change what the user wrote.
|
||||
//
|
||||
UrlSafety.normalizeForKey = (raw) => {
|
||||
const url = UrlSafety.parse(raw)
|
||||
if (!url) { return null }
|
||||
|
||||
url.hash = ''
|
||||
url.username = ''
|
||||
url.password = ''
|
||||
|
||||
// Trailing dot on the host is legal DNS but splits the cache key.
|
||||
if (url.hostname.endsWith('.')) {
|
||||
url.hostname = url.hostname.slice(0, -1)
|
||||
}
|
||||
url.hostname = url.hostname.toLowerCase()
|
||||
|
||||
// new URL already strips the default port, but be explicit.
|
||||
if ((url.protocol === 'http:' && url.port === '80') ||
|
||||
(url.protocol === 'https:' && url.port === '443')) {
|
||||
url.port = ''
|
||||
}
|
||||
|
||||
for (const key of [...url.searchParams.keys()]) {
|
||||
if (TRACKING_PARAMS.some(pattern => pattern.test(key))) {
|
||||
url.searchParams.delete(key)
|
||||
}
|
||||
}
|
||||
url.searchParams.sort()
|
||||
|
||||
if (url.pathname === '') { url.pathname = '/' }
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
// sha256 of the normalised href. This is scrape_cache.url_hash - char(64),
|
||||
// because a UNIQUE index on varchar(1000) utf8mb4 would be 4000 bytes and
|
||||
// blow InnoDB's 3072-byte key limit.
|
||||
UrlSafety.urlKey = (raw) => {
|
||||
const url = UrlSafety.normalizeForKey(raw)
|
||||
if (!url) { return null }
|
||||
return crypto.createHash('sha256').update(url.href).digest('hex')
|
||||
}
|
||||
|
||||
//
|
||||
// True when two URLs address the same resource modulo scheme, "www." and a
|
||||
// trailing slash.
|
||||
//
|
||||
// Load-bearing: scanTextForWebsites diffs the URLs currently in a note against
|
||||
// the URLs already stored as attachments. Raw string equality would treat
|
||||
// http://foo.com and https://foo.com as different resources and delete-then-
|
||||
// rescrape the row on every save.
|
||||
//
|
||||
UrlSafety.sameResource = (a, b) => {
|
||||
const left = UrlSafety.normalizeForKey(a)
|
||||
const right = UrlSafety.normalizeForKey(b)
|
||||
if (!left || !right) { return false }
|
||||
|
||||
const host = (url) => url.hostname.replace(/^www\./, '')
|
||||
const path = (url) => url.pathname.replace(/\/+$/, '')
|
||||
|
||||
return host(left) === host(right) &&
|
||||
path(left) === path(right) &&
|
||||
left.search === right.search
|
||||
}
|
||||
|
||||
UrlSafety.registrableHost = (hostname) => {
|
||||
return String(hostname || '').toLowerCase().replace(/^www\./, '').replace(/\.$/, '')
|
||||
}
|
||||
|
||||
//
|
||||
// Private / non-routable address detection.
|
||||
//
|
||||
// Covers the ranges that matter for SSRF against this app: loopback, RFC1918,
|
||||
// link-local (including the 169.254.169.254 cloud metadata endpoint), CGNAT,
|
||||
// and the IPv6 equivalents.
|
||||
//
|
||||
UrlSafety.isPrivateIp = (address) => {
|
||||
if (!address) { return true }
|
||||
|
||||
const version = net.isIP(address)
|
||||
if (version === 0) { return true } // not an IP - caller resolves first
|
||||
|
||||
if (version === 4) { return isPrivateIpv4(address) }
|
||||
return isPrivateIpv6(address)
|
||||
}
|
||||
|
||||
const isPrivateIpv4 = (address) => {
|
||||
const parts = address.split('.').map(Number)
|
||||
if (parts.length !== 4 || parts.some(n => !Number.isInteger(n) || n < 0 || n > 255)) {
|
||||
return true
|
||||
}
|
||||
const [a, b] = parts
|
||||
|
||||
if (a === 0) { return true } // 0.0.0.0/8 "this host"
|
||||
if (a === 10) { return true } // RFC1918
|
||||
if (a === 127) { return true } // loopback
|
||||
if (a === 169 && b === 254) { return true } // link-local + metadata
|
||||
if (a === 172 && b >= 16 && b <= 31) { return true } // RFC1918
|
||||
if (a === 192 && b === 168) { return true } // RFC1918
|
||||
if (a === 192 && b === 0) { return true } // 192.0.0/24 + 192.0.2/24
|
||||
if (a === 198 && (b === 18 || b === 19)) { return true } // benchmarking
|
||||
if (a === 100 && b >= 64 && b <= 127) { return true } // CGNAT
|
||||
if (a === 198 && b === 51) { return true } // TEST-NET-2
|
||||
if (a === 203 && b === 0) { return true } // TEST-NET-3
|
||||
if (a >= 224) { return true } // multicast + reserved
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
const isPrivateIpv6 = (address) => {
|
||||
const lower = address.toLowerCase().replace(/^\[|\]$/g, '')
|
||||
|
||||
// Unwrap v4-mapped (::ffff:127.0.0.1) and NAT64 (64:ff9b::/96) before
|
||||
// judging, otherwise ::ffff:127.0.0.1 reads as "some v6 address".
|
||||
const mapped = lower.match(/^(?:::ffff:|64:ff9b::)(\d+\.\d+\.\d+\.\d+)$/)
|
||||
if (mapped) { return isPrivateIpv4(mapped[1]) }
|
||||
|
||||
if (lower === '::' || lower === '::1') { return true } // unspecified, loopback
|
||||
if (/^f[cd]/.test(lower)) { return true } // fc00::/7 unique-local
|
||||
if (/^fe[89ab]/.test(lower)) { return true } // fe80::/10 link-local
|
||||
if (/^ff/.test(lower)) { return true } // multicast
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
//
|
||||
// The gate. Throws SafetyError; resolves to the vetted addresses on success.
|
||||
//
|
||||
// Known limitation (documented, accepted): this resolves DNS and then hands the
|
||||
// hostname to fetch, which resolves again. A hostile 0-TTL record can answer
|
||||
// differently the second time. Closing that properly needs a custom undici
|
||||
// Agent with connect.lookup - deliberately deferred, see the plan's Phase 5.
|
||||
// Residual risk is blind SSRF only: no credentials are sent and no response
|
||||
// body is returned to whoever planted the URL.
|
||||
//
|
||||
UrlSafety.assertFetchable = async (raw) => {
|
||||
const url = raw instanceof URL ? raw : UrlSafety.parse(raw)
|
||||
|
||||
if (!url) {
|
||||
throw new SafetyError('bad_url', `Not a fetchable URL: ${raw}`)
|
||||
}
|
||||
if (!FETCHABLE_SCHEMES.has(url.protocol)) {
|
||||
throw new SafetyError('bad_scheme', `Refusing scheme ${url.protocol}`)
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new SafetyError('credentials_in_url', 'Refusing URL with embedded credentials')
|
||||
}
|
||||
|
||||
const hostname = url.hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase()
|
||||
|
||||
if (!hostname) {
|
||||
throw new SafetyError('bad_url', 'URL has no host')
|
||||
}
|
||||
if (hostname === 'localhost' || BLOCKED_SUFFIXES.some(suffix => hostname.endsWith(suffix))) {
|
||||
throw new SafetyError('blocked_host', `Refusing internal host ${hostname}`)
|
||||
}
|
||||
|
||||
// Literal IP in the URL - no DNS needed.
|
||||
if (net.isIP(hostname)) {
|
||||
if (UrlSafety.isPrivateIp(hostname)) {
|
||||
throw new SafetyError('private_ip', `Refusing non-routable address ${hostname}`)
|
||||
}
|
||||
return [hostname]
|
||||
}
|
||||
|
||||
let resolved = []
|
||||
try {
|
||||
resolved = await dns.lookup(hostname, { all: true, verbatim: true })
|
||||
} catch (error) {
|
||||
throw new SafetyError('dns_failed', `Could not resolve ${hostname}: ${error.code || error.message}`)
|
||||
}
|
||||
|
||||
if (!resolved.length) {
|
||||
throw new SafetyError('dns_failed', `No addresses for ${hostname}`)
|
||||
}
|
||||
|
||||
// Every answer must be public. One private address is enough to refuse -
|
||||
// otherwise a round-robin record gets us some of the time.
|
||||
for (const entry of resolved) {
|
||||
if (UrlSafety.isPrivateIp(entry.address)) {
|
||||
throw new SafetyError('private_ip', `${hostname} resolves to non-routable ${entry.address}`)
|
||||
}
|
||||
}
|
||||
|
||||
return resolved.map(entry => entry.address)
|
||||
}
|
||||
Reference in new issue
Block a user