Adding scrape files for testing

server/config/scrape.js

@@ -0,0 +1,70 @@
+//
+// All scraper tunables in one place, so nothing downstream is a magic number.
+//
+// Env overrides exist for the two kill switches only. Everything else is a
+// constant on purpose: the politeness numbers are the difference between
+// looking like a browser and looking like a bot, and they should not drift
+// per-environment without a code change.
+//
+
+const isTest = process.env.NODE_ENV === 'test'
+
+const config = {
+	// Kill switches
+	enabled: process.env.SCRAPE_ENABLED !== '0',
+	// The background worker is off under Jest so tests drive the queue
+	// deterministically via ScrapeQueue.drain() instead of racing it.
+	workerEnabled: process.env.SCRAPE_WORKER ? process.env.SCRAPE_WORKER === '1' : !isTest,
+
+	// Timeouts
+	pageTimeoutMs: 8000,     // single HTTP request
+	totalBudgetMs: 25000,    // all hops + retries for one URL
+	imageTimeoutMs: 10000,
+
+	// Size caps. Content-Length is optional and can lie, so these are also
+	// enforced while streaming the body.
+	maxHtmlBytes: 2 * 1024 * 1024,
+	maxImageBytes: 5 * 1024 * 1024,
+	maxRobotsBytes: 100 * 1024,
+
+	// Redirects
+	maxRedirects: 5,
+
+	// Retries. Only for transient classes - a 404 is an answer, not a failure.
+	retries: 2,
+	backoffBaseMs: 500,
+	backoffMaxMs: 8000,
+	maxRetryAfterMs: 60 * 1000,
+
+	// Politeness. One request at a time per host, spaced out, is what actually
+	// stops us looking like a crawler.
+	maxConcurrentFetches: 4,
+	perHostConcurrency: 1,
+	perHostMinSpacingMs: 1000,
+	perHostMaxSpacingMs: 3000,
+	maxCrawlDelayMs: 30 * 1000,
+
+	// Backpressure
+	maxQueueLength: 500,
+	maxPendingPerUser: 50,
+
+	// Identity. Stable salt, so a host always sees the same browser across
+	// restarts. Changing this re-rolls every host's assigned profile.
+	profileSalt: 'ss-scrape-v1',
+	robotsUaToken: 'SolidScribeBot',
+	robotsTtlMs: 24 * 60 * 60 * 1000,
+	robotsNegativeTtlMs: 60 * 60 * 1000,
+	robotsErrorTtlMs: 10 * 60 * 1000,
+	robotsMaxOrigins: 2000,
+
+	// Parsing / storage
+	maxSearchTextBytes: 8000,
+	maxKeywordSourceChars: 6000,
+	maxUrlLength: 1000, // matches attachment.url varchar(1000)
+
+	// staticFiles is served from our own origin via /api/static, so an SVG
+	// with an inline script would be same-origin. Off by default.
+	allowSvgThumbnails: false,
+}
+
+module.exports = config