//
// All scraper tunables in one place, so nothing downstream is a magic number.
//
// Env overrides exist for the two kill switches only. Everything else is a
// constant on purpose: the politeness numbers are the difference between
// looking like a browser and looking like a bot, and they should not drift
// per-environment without a code change.
//

const isTest = process.env.NODE_ENV === 'test'

const config = {
	// Kill switches
	enabled: process.env.SCRAPE_ENABLED !== '0',
	// The background worker is off under Jest so tests drive the queue
	// deterministically via ScrapeQueue.drain() instead of racing it.
	workerEnabled: process.env.SCRAPE_WORKER ? process.env.SCRAPE_WORKER === '1' : !isTest,

	// Timeouts
	pageTimeoutMs: 8000,     // single HTTP request
	totalBudgetMs: 25000,    // all hops + retries for one URL
	imageTimeoutMs: 10000,

	// Size caps. Content-Length is optional and can lie, so these are also
	// enforced while streaming the body.
	maxHtmlBytes: 2 * 1024 * 1024,
	maxImageBytes: 5 * 1024 * 1024,
	maxRobotsBytes: 100 * 1024,

	// Redirects
	maxRedirects: 5,

	// Retries. Only for transient classes - a 404 is an answer, not a failure.
	retries: 2,
	backoffBaseMs: 500,
	backoffMaxMs: 8000,
	maxRetryAfterMs: 60 * 1000,

	// Politeness. One request at a time per host, spaced out, is what actually
	// stops us looking like a crawler.
	maxConcurrentFetches: 4,
	perHostConcurrency: 1,
	perHostMinSpacingMs: 1000,
	perHostMaxSpacingMs: 3000,
	maxCrawlDelayMs: 30 * 1000,

	// Backpressure
	maxQueueLength: 500,
	maxPendingPerUser: 50,

	// Identity. Stable salt, so a host always sees the same browser across
	// restarts. Changing this re-rolls every host's assigned profile.
	profileSalt: 'ss-scrape-v1',
	robotsUaToken: 'SolidScribeBot',
	robotsTtlMs: 24 * 60 * 60 * 1000,
	robotsNegativeTtlMs: 60 * 60 * 1000,
	robotsErrorTtlMs: 10 * 60 * 1000,
	robotsMaxOrigins: 2000,

	// Parsing / storage
	maxSearchTextBytes: 8000,
	maxKeywordSourceChars: 6000,
	maxUrlLength: 1000, // matches attachment.url varchar(1000)

	// staticFiles is served from our own origin via /api/static, so an SVG
	// with an inline script would be same-origin. Off by default.
	allowSvgThumbnails: false,
}

module.exports = config