196 lines
6.3 KiB
JavaScript
196 lines
6.3 KiB
JavaScript
//
|
|
// Browser identities for outbound fetches.
|
|
//
|
|
// The old code sent a single hardcoded Googlebot User-Agent. That was removed
|
|
// deliberately: Googlebot is verifiable by reverse DNS, so the claim is trivially
|
|
// disproven, and sites that check it block the source IP outright. Some also
|
|
// serve crawlers a stripped-down page.
|
|
//
|
|
// What replaces it is an honest, *coherent* browser identity. Coherence is the
|
|
// point - a Chrome User-Agent shipping Firefox's Accept header, or sec-ch-ua on
|
|
// a Safari request, is a stronger bot signal than any UA string. So each profile
|
|
// carries the complete header set its family really sends, and nothing mixes.
|
|
//
|
|
// Profiles are assigned stably per host, not randomly per request: a site should
|
|
// see one returning browser, not a rotating cast.
|
|
//
|
|
|
|
const config = require('@config/scrape')
|
|
|
|
let BrowserProfiles = module.exports = {}
|
|
|
|
const CHROME_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'
|
|
const FIREFOX_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
|
|
const SAFARI_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
|
|
|
|
const PROFILES = BrowserProfiles.PROFILES = [
|
|
{
|
|
id: 'chrome-win',
|
|
family: 'chrome',
|
|
ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
|
accept: CHROME_ACCEPT,
|
|
acceptLanguage: 'en-US,en;q=0.9',
|
|
secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
|
|
secChUaMobile: '?0',
|
|
secChUaPlatform: '"Windows"',
|
|
},
|
|
{
|
|
id: 'chrome-mac',
|
|
family: 'chrome',
|
|
ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
|
accept: CHROME_ACCEPT,
|
|
acceptLanguage: 'en-US,en;q=0.9',
|
|
secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
|
|
secChUaMobile: '?0',
|
|
secChUaPlatform: '"macOS"',
|
|
},
|
|
{
|
|
id: 'firefox-win',
|
|
family: 'firefox',
|
|
ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
|
accept: FIREFOX_ACCEPT,
|
|
acceptLanguage: 'en-US,en;q=0.5',
|
|
},
|
|
{
|
|
id: 'firefox-linux',
|
|
family: 'firefox',
|
|
ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
|
accept: FIREFOX_ACCEPT,
|
|
acceptLanguage: 'en-US,en;q=0.5',
|
|
},
|
|
{
|
|
id: 'safari-mac',
|
|
family: 'safari',
|
|
ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
|
|
accept: SAFARI_ACCEPT,
|
|
acceptLanguage: 'en-US,en;q=0.9',
|
|
},
|
|
]
|
|
|
|
// FNV-1a. Not cryptographic - we only need a stable, well-spread bucket.
|
|
const fnv1a = (input) => {
|
|
let hash = 0x811c9dc5
|
|
for (let i = 0; i < input.length; i++) {
|
|
hash ^= input.charCodeAt(i)
|
|
hash = Math.imul(hash, 0x01000193) >>> 0
|
|
}
|
|
return hash >>> 0
|
|
}
|
|
BrowserProfiles.__fnv1a = fnv1a
|
|
|
|
//
|
|
// Same host always gets the same profile, across requests and across restarts
|
|
// (the salt is a constant in config, not a per-boot random).
|
|
//
|
|
BrowserProfiles.profileForHost = (hostname) => {
|
|
const key = require('@helpers/UrlSafety').registrableHost(hostname) + config.profileSalt
|
|
return PROFILES[fnv1a(key) % PROFILES.length]
|
|
}
|
|
|
|
BrowserProfiles.profileById = (id) => PROFILES.find(profile => profile.id === id) || PROFILES[0]
|
|
|
|
//
|
|
// Sec-Fetch-Site is relative: "none" for a URL typed into the bar, "same-origin"
|
|
// when the referrer shares the origin, "cross-site" otherwise. Getting this
|
|
// wrong while sending a Referer is exactly the kind of incoherence that flags.
|
|
//
|
|
const fetchSite = (targetUrl, referer) => {
|
|
if (!referer) { return 'none' }
|
|
try {
|
|
const target = new URL(targetUrl)
|
|
const from = new URL(referer)
|
|
if (target.origin === from.origin) { return 'same-origin' }
|
|
if (UrlSafetyHost(target) === UrlSafetyHost(from)) { return 'same-site' }
|
|
return 'cross-site'
|
|
} catch (error) {
|
|
return 'none'
|
|
}
|
|
}
|
|
|
|
const UrlSafetyHost = (url) => require('@helpers/UrlSafety').registrableHost(url.hostname)
|
|
|
|
const applyChromiumHints = (headers, profile) => {
|
|
if (profile.family !== 'chrome') { return headers }
|
|
headers['sec-ch-ua'] = profile.secChUa
|
|
headers['sec-ch-ua-mobile'] = profile.secChUaMobile
|
|
headers['sec-ch-ua-platform'] = profile.secChUaPlatform
|
|
return headers
|
|
}
|
|
|
|
//
|
|
// Headers for a top-level document navigation.
|
|
//
|
|
BrowserProfiles.documentHeaders = (profile, { url, referer } = {}) => {
|
|
const headers = {
|
|
'User-Agent': profile.ua,
|
|
'Accept': profile.accept,
|
|
'Accept-Language': profile.acceptLanguage,
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
}
|
|
|
|
// Safari sends neither sec-ch-ua nor Sec-Fetch-*; Firefox sends Sec-Fetch-*
|
|
// but no client hints; Chromium sends both.
|
|
if (profile.family !== 'safari') {
|
|
headers['Sec-Fetch-Dest'] = 'document'
|
|
headers['Sec-Fetch-Mode'] = 'navigate'
|
|
headers['Sec-Fetch-Site'] = fetchSite(url, referer)
|
|
headers['Sec-Fetch-User'] = '?1'
|
|
headers['Upgrade-Insecure-Requests'] = '1'
|
|
}
|
|
|
|
applyChromiumHints(headers, profile)
|
|
|
|
if (referer) { headers['Referer'] = referer }
|
|
|
|
return headers
|
|
}
|
|
|
|
//
|
|
// Headers for a subresource image load, as issued from the page it was found on.
|
|
//
|
|
BrowserProfiles.imageHeaders = (profile, { url, pageUrl } = {}) => {
|
|
const headers = {
|
|
'User-Agent': profile.ua,
|
|
'Accept': profile.family === 'chrome'
|
|
? 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8'
|
|
: 'image/avif,image/webp,*/*',
|
|
'Accept-Language': profile.acceptLanguage,
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
}
|
|
|
|
if (profile.family !== 'safari') {
|
|
headers['Sec-Fetch-Dest'] = 'image'
|
|
headers['Sec-Fetch-Mode'] = 'no-cors'
|
|
headers['Sec-Fetch-Site'] = fetchSite(url, pageUrl)
|
|
}
|
|
|
|
applyChromiumHints(headers, profile)
|
|
|
|
if (pageUrl) { headers['Referer'] = pageUrl }
|
|
|
|
return headers
|
|
}
|
|
|
|
//
|
|
// Headers for robots.txt. Same identity as the page fetch, but it is not a
|
|
// navigation - a browser would never send Sec-Fetch-Dest: document for this.
|
|
//
|
|
BrowserProfiles.robotsHeaders = (profile) => {
|
|
const headers = {
|
|
'User-Agent': profile.ua,
|
|
'Accept': 'text/plain,*/*;q=0.8',
|
|
'Accept-Language': profile.acceptLanguage,
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
}
|
|
|
|
if (profile.family !== 'safari') {
|
|
headers['Sec-Fetch-Dest'] = 'empty'
|
|
headers['Sec-Fetch-Mode'] = 'no-cors'
|
|
headers['Sec-Fetch-Site'] = 'none'
|
|
}
|
|
|
|
applyChromiumHints(headers, profile)
|
|
|
|
return headers
|
|
}
|