71 lines
2.2 KiB
JavaScript
71 lines
2.2 KiB
JavaScript
//
|
|
// All scraper tunables in one place, so nothing downstream is a magic number.
|
|
//
|
|
// Env overrides exist for the two kill switches only. Everything else is a
|
|
// constant on purpose: the politeness numbers are the difference between
|
|
// looking like a browser and looking like a bot, and they should not drift
|
|
// per-environment without a code change.
|
|
//
|
|
|
|
const isTest = process.env.NODE_ENV === 'test'
|
|
|
|
const config = {
|
|
// Kill switches
|
|
enabled: process.env.SCRAPE_ENABLED !== '0',
|
|
// The background worker is off under Jest so tests drive the queue
|
|
// deterministically via ScrapeQueue.drain() instead of racing it.
|
|
workerEnabled: process.env.SCRAPE_WORKER ? process.env.SCRAPE_WORKER === '1' : !isTest,
|
|
|
|
// Timeouts
|
|
pageTimeoutMs: 8000, // single HTTP request
|
|
totalBudgetMs: 25000, // all hops + retries for one URL
|
|
imageTimeoutMs: 10000,
|
|
|
|
// Size caps. Content-Length is optional and can lie, so these are also
|
|
// enforced while streaming the body.
|
|
maxHtmlBytes: 2 * 1024 * 1024,
|
|
maxImageBytes: 5 * 1024 * 1024,
|
|
maxRobotsBytes: 100 * 1024,
|
|
|
|
// Redirects
|
|
maxRedirects: 5,
|
|
|
|
// Retries. Only for transient classes - a 404 is an answer, not a failure.
|
|
retries: 2,
|
|
backoffBaseMs: 500,
|
|
backoffMaxMs: 8000,
|
|
maxRetryAfterMs: 60 * 1000,
|
|
|
|
// Politeness. One request at a time per host, spaced out, is what actually
|
|
// stops us looking like a crawler.
|
|
maxConcurrentFetches: 4,
|
|
perHostConcurrency: 1,
|
|
perHostMinSpacingMs: 1000,
|
|
perHostMaxSpacingMs: 3000,
|
|
maxCrawlDelayMs: 30 * 1000,
|
|
|
|
// Backpressure
|
|
maxQueueLength: 500,
|
|
maxPendingPerUser: 50,
|
|
|
|
// Identity. Stable salt, so a host always sees the same browser across
|
|
// restarts. Changing this re-rolls every host's assigned profile.
|
|
profileSalt: 'ss-scrape-v1',
|
|
robotsUaToken: 'SolidScribeBot',
|
|
robotsTtlMs: 24 * 60 * 60 * 1000,
|
|
robotsNegativeTtlMs: 60 * 60 * 1000,
|
|
robotsErrorTtlMs: 10 * 60 * 1000,
|
|
robotsMaxOrigins: 2000,
|
|
|
|
// Parsing / storage
|
|
maxSearchTextBytes: 8000,
|
|
maxKeywordSourceChars: 6000,
|
|
maxUrlLength: 1000, // matches attachment.url varchar(1000)
|
|
|
|
// staticFiles is served from our own origin via /api/static, so an SVG
|
|
// with an inline script would be same-origin. Off by default.
|
|
allowSvgThumbnails: false,
|
|
}
|
|
|
|
module.exports = config
|