Adding scrape files for testing
This commit is contained in:
Max committed
1 parent
7d255fe687
commit
8d60aae65c
40 files changed
+4423
No files matched your search
@@ -0,0 +1,195 @@
|
||||
//
|
||||
// Browser identities for outbound fetches.
|
||||
//
|
||||
// The old code sent a single hardcoded Googlebot User-Agent. That was removed
|
||||
// deliberately: Googlebot is verifiable by reverse DNS, so the claim is trivially
|
||||
// disproven, and sites that check it block the source IP outright. Some also
|
||||
// serve crawlers a stripped-down page.
|
||||
//
|
||||
// What replaces it is an honest, *coherent* browser identity. Coherence is the
|
||||
// point - a Chrome User-Agent shipping Firefox's Accept header, or sec-ch-ua on
|
||||
// a Safari request, is a stronger bot signal than any UA string. So each profile
|
||||
// carries the complete header set its family really sends, and nothing mixes.
|
||||
//
|
||||
// Profiles are assigned stably per host, not randomly per request: a site should
|
||||
// see one returning browser, not a rotating cast.
|
||||
//
|
||||
|
||||
const config = require('@config/scrape')
|
||||
|
||||
let BrowserProfiles = module.exports = {}
|
||||
|
||||
const CHROME_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'
|
||||
const FIREFOX_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
|
||||
const SAFARI_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
|
||||
|
||||
const PROFILES = BrowserProfiles.PROFILES = [
|
||||
{
|
||||
id: 'chrome-win',
|
||||
family: 'chrome',
|
||||
ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
||||
accept: CHROME_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.9',
|
||||
secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
|
||||
secChUaMobile: '?0',
|
||||
secChUaPlatform: '"Windows"',
|
||||
},
|
||||
{
|
||||
id: 'chrome-mac',
|
||||
family: 'chrome',
|
||||
ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
||||
accept: CHROME_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.9',
|
||||
secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
|
||||
secChUaMobile: '?0',
|
||||
secChUaPlatform: '"macOS"',
|
||||
},
|
||||
{
|
||||
id: 'firefox-win',
|
||||
family: 'firefox',
|
||||
ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
||||
accept: FIREFOX_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.5',
|
||||
},
|
||||
{
|
||||
id: 'firefox-linux',
|
||||
family: 'firefox',
|
||||
ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
||||
accept: FIREFOX_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.5',
|
||||
},
|
||||
{
|
||||
id: 'safari-mac',
|
||||
family: 'safari',
|
||||
ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
|
||||
accept: SAFARI_ACCEPT,
|
||||
acceptLanguage: 'en-US,en;q=0.9',
|
||||
},
|
||||
]
|
||||
|
||||
// FNV-1a. Not cryptographic - we only need a stable, well-spread bucket.
|
||||
const fnv1a = (input) => {
|
||||
let hash = 0x811c9dc5
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
hash ^= input.charCodeAt(i)
|
||||
hash = Math.imul(hash, 0x01000193) >>> 0
|
||||
}
|
||||
return hash >>> 0
|
||||
}
|
||||
BrowserProfiles.__fnv1a = fnv1a
|
||||
|
||||
//
|
||||
// Same host always gets the same profile, across requests and across restarts
|
||||
// (the salt is a constant in config, not a per-boot random).
|
||||
//
|
||||
BrowserProfiles.profileForHost = (hostname) => {
|
||||
const key = require('@helpers/UrlSafety').registrableHost(hostname) + config.profileSalt
|
||||
return PROFILES[fnv1a(key) % PROFILES.length]
|
||||
}
|
||||
|
||||
BrowserProfiles.profileById = (id) => PROFILES.find(profile => profile.id === id) || PROFILES[0]
|
||||
|
||||
//
|
||||
// Sec-Fetch-Site is relative: "none" for a URL typed into the bar, "same-origin"
|
||||
// when the referrer shares the origin, "cross-site" otherwise. Getting this
|
||||
// wrong while sending a Referer is exactly the kind of incoherence that flags.
|
||||
//
|
||||
const fetchSite = (targetUrl, referer) => {
|
||||
if (!referer) { return 'none' }
|
||||
try {
|
||||
const target = new URL(targetUrl)
|
||||
const from = new URL(referer)
|
||||
if (target.origin === from.origin) { return 'same-origin' }
|
||||
if (UrlSafetyHost(target) === UrlSafetyHost(from)) { return 'same-site' }
|
||||
return 'cross-site'
|
||||
} catch (error) {
|
||||
return 'none'
|
||||
}
|
||||
}
|
||||
|
||||
const UrlSafetyHost = (url) => require('@helpers/UrlSafety').registrableHost(url.hostname)
|
||||
|
||||
const applyChromiumHints = (headers, profile) => {
|
||||
if (profile.family !== 'chrome') { return headers }
|
||||
headers['sec-ch-ua'] = profile.secChUa
|
||||
headers['sec-ch-ua-mobile'] = profile.secChUaMobile
|
||||
headers['sec-ch-ua-platform'] = profile.secChUaPlatform
|
||||
return headers
|
||||
}
|
||||
|
||||
//
|
||||
// Headers for a top-level document navigation.
|
||||
//
|
||||
BrowserProfiles.documentHeaders = (profile, { url, referer } = {}) => {
|
||||
const headers = {
|
||||
'User-Agent': profile.ua,
|
||||
'Accept': profile.accept,
|
||||
'Accept-Language': profile.acceptLanguage,
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
}
|
||||
|
||||
// Safari sends neither sec-ch-ua nor Sec-Fetch-*; Firefox sends Sec-Fetch-*
|
||||
// but no client hints; Chromium sends both.
|
||||
if (profile.family !== 'safari') {
|
||||
headers['Sec-Fetch-Dest'] = 'document'
|
||||
headers['Sec-Fetch-Mode'] = 'navigate'
|
||||
headers['Sec-Fetch-Site'] = fetchSite(url, referer)
|
||||
headers['Sec-Fetch-User'] = '?1'
|
||||
headers['Upgrade-Insecure-Requests'] = '1'
|
||||
}
|
||||
|
||||
applyChromiumHints(headers, profile)
|
||||
|
||||
if (referer) { headers['Referer'] = referer }
|
||||
|
||||
return headers
|
||||
}
|
||||
|
||||
//
|
||||
// Headers for a subresource image load, as issued from the page it was found on.
|
||||
//
|
||||
BrowserProfiles.imageHeaders = (profile, { url, pageUrl } = {}) => {
|
||||
const headers = {
|
||||
'User-Agent': profile.ua,
|
||||
'Accept': profile.family === 'chrome'
|
||||
? 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8'
|
||||
: 'image/avif,image/webp,*/*',
|
||||
'Accept-Language': profile.acceptLanguage,
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
}
|
||||
|
||||
if (profile.family !== 'safari') {
|
||||
headers['Sec-Fetch-Dest'] = 'image'
|
||||
headers['Sec-Fetch-Mode'] = 'no-cors'
|
||||
headers['Sec-Fetch-Site'] = fetchSite(url, pageUrl)
|
||||
}
|
||||
|
||||
applyChromiumHints(headers, profile)
|
||||
|
||||
if (pageUrl) { headers['Referer'] = pageUrl }
|
||||
|
||||
return headers
|
||||
}
|
||||
|
||||
//
|
||||
// Headers for robots.txt. Same identity as the page fetch, but it is not a
|
||||
// navigation - a browser would never send Sec-Fetch-Dest: document for this.
|
||||
//
|
||||
BrowserProfiles.robotsHeaders = (profile) => {
|
||||
const headers = {
|
||||
'User-Agent': profile.ua,
|
||||
'Accept': 'text/plain,*/*;q=0.8',
|
||||
'Accept-Language': profile.acceptLanguage,
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
}
|
||||
|
||||
if (profile.family !== 'safari') {
|
||||
headers['Sec-Fetch-Dest'] = 'empty'
|
||||
headers['Sec-Fetch-Mode'] = 'no-cors'
|
||||
headers['Sec-Fetch-Site'] = 'none'
|
||||
}
|
||||
|
||||
applyChromiumHints(headers, profile)
|
||||
|
||||
return headers
|
||||
}
|
||||
Reference in new issue
Block a user