Adding scrape files for testing
This commit is contained in:
Max committed
1 parent
7d255fe687
commit
8d60aae65c
40 files changed
+4423
No files matched your search
@@ -0,0 +1,246 @@
|
||||
//
|
||||
// URL normalisation and the SSRF gate.
|
||||
//
|
||||
// Scraped URLs come from untrusted note text and from the /pushmebaby query
|
||||
// param, so every URL is validated before a socket is opened - and again after
|
||||
// every redirect hop, which is the hop that actually matters (a public URL that
|
||||
// 302s to 127.0.0.1 is the classic bypass).
|
||||
//
|
||||
// Pure except for assertFetchable, which resolves DNS.
|
||||
//
|
||||
|
||||
const crypto = require('crypto')
|
||||
const dns = require('dns').promises
|
||||
const net = require('net')
|
||||
const config = require('@config/scrape')
|
||||
|
||||
let UrlSafety = module.exports = {}
|
||||
|
||||
const FETCHABLE_SCHEMES = new Set(['http:', 'https:'])
|
||||
|
||||
// Hostname suffixes that never belong to a public site.
|
||||
const BLOCKED_SUFFIXES = ['.localhost', '.local', '.internal', '.home.arpa', '.lan']
|
||||
|
||||
// Query params that change nothing about the page but would fragment the cache.
|
||||
const TRACKING_PARAMS = [
|
||||
/^utm_/i, /^fbclid$/i, /^gclid$/i, /^dclid$/i, /^msclkid$/i,
|
||||
/^mc_(e|c)id$/i, /^igshid$/i, /^ref_src$/i, /^_hs(enc|mi)$/i, /^vero_/i,
|
||||
]
|
||||
|
||||
class SafetyError extends Error {
|
||||
constructor(reason, message) {
|
||||
super(message || reason)
|
||||
this.name = 'SafetyError'
|
||||
this.reason = reason
|
||||
}
|
||||
}
|
||||
UrlSafety.SafetyError = SafetyError
|
||||
|
||||
//
|
||||
// Parse to a URL object, or null. Does NOT mutate anything that would change
|
||||
// which resource is addressed - callers may store the result.
|
||||
//
|
||||
UrlSafety.parse = (raw, base) => {
|
||||
if (!raw || typeof raw !== 'string') { return null }
|
||||
|
||||
let url = null
|
||||
try {
|
||||
url = new URL(raw.trim(), base || undefined)
|
||||
} catch (error) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (!FETCHABLE_SCHEMES.has(url.protocol)) { return null }
|
||||
if (url.href.length > config.maxUrlLength) { return null }
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
//
|
||||
// Canonical form used ONLY for cache keys. Never store this - it deliberately
|
||||
// discards tracking params and case, which would change what the user wrote.
|
||||
//
|
||||
UrlSafety.normalizeForKey = (raw) => {
|
||||
const url = UrlSafety.parse(raw)
|
||||
if (!url) { return null }
|
||||
|
||||
url.hash = ''
|
||||
url.username = ''
|
||||
url.password = ''
|
||||
|
||||
// Trailing dot on the host is legal DNS but splits the cache key.
|
||||
if (url.hostname.endsWith('.')) {
|
||||
url.hostname = url.hostname.slice(0, -1)
|
||||
}
|
||||
url.hostname = url.hostname.toLowerCase()
|
||||
|
||||
// new URL already strips the default port, but be explicit.
|
||||
if ((url.protocol === 'http:' && url.port === '80') ||
|
||||
(url.protocol === 'https:' && url.port === '443')) {
|
||||
url.port = ''
|
||||
}
|
||||
|
||||
for (const key of [...url.searchParams.keys()]) {
|
||||
if (TRACKING_PARAMS.some(pattern => pattern.test(key))) {
|
||||
url.searchParams.delete(key)
|
||||
}
|
||||
}
|
||||
url.searchParams.sort()
|
||||
|
||||
if (url.pathname === '') { url.pathname = '/' }
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
// sha256 of the normalised href. This is scrape_cache.url_hash - char(64),
|
||||
// because a UNIQUE index on varchar(1000) utf8mb4 would be 4000 bytes and
|
||||
// blow InnoDB's 3072-byte key limit.
|
||||
UrlSafety.urlKey = (raw) => {
|
||||
const url = UrlSafety.normalizeForKey(raw)
|
||||
if (!url) { return null }
|
||||
return crypto.createHash('sha256').update(url.href).digest('hex')
|
||||
}
|
||||
|
||||
//
|
||||
// True when two URLs address the same resource modulo scheme, "www." and a
|
||||
// trailing slash.
|
||||
//
|
||||
// Load-bearing: scanTextForWebsites diffs the URLs currently in a note against
|
||||
// the URLs already stored as attachments. Raw string equality would treat
|
||||
// http://foo.com and https://foo.com as different resources and delete-then-
|
||||
// rescrape the row on every save.
|
||||
//
|
||||
UrlSafety.sameResource = (a, b) => {
|
||||
const left = UrlSafety.normalizeForKey(a)
|
||||
const right = UrlSafety.normalizeForKey(b)
|
||||
if (!left || !right) { return false }
|
||||
|
||||
const host = (url) => url.hostname.replace(/^www\./, '')
|
||||
const path = (url) => url.pathname.replace(/\/+$/, '')
|
||||
|
||||
return host(left) === host(right) &&
|
||||
path(left) === path(right) &&
|
||||
left.search === right.search
|
||||
}
|
||||
|
||||
UrlSafety.registrableHost = (hostname) => {
|
||||
return String(hostname || '').toLowerCase().replace(/^www\./, '').replace(/\.$/, '')
|
||||
}
|
||||
|
||||
//
|
||||
// Private / non-routable address detection.
|
||||
//
|
||||
// Covers the ranges that matter for SSRF against this app: loopback, RFC1918,
|
||||
// link-local (including the 169.254.169.254 cloud metadata endpoint), CGNAT,
|
||||
// and the IPv6 equivalents.
|
||||
//
|
||||
UrlSafety.isPrivateIp = (address) => {
|
||||
if (!address) { return true }
|
||||
|
||||
const version = net.isIP(address)
|
||||
if (version === 0) { return true } // not an IP - caller resolves first
|
||||
|
||||
if (version === 4) { return isPrivateIpv4(address) }
|
||||
return isPrivateIpv6(address)
|
||||
}
|
||||
|
||||
const isPrivateIpv4 = (address) => {
|
||||
const parts = address.split('.').map(Number)
|
||||
if (parts.length !== 4 || parts.some(n => !Number.isInteger(n) || n < 0 || n > 255)) {
|
||||
return true
|
||||
}
|
||||
const [a, b] = parts
|
||||
|
||||
if (a === 0) { return true } // 0.0.0.0/8 "this host"
|
||||
if (a === 10) { return true } // RFC1918
|
||||
if (a === 127) { return true } // loopback
|
||||
if (a === 169 && b === 254) { return true } // link-local + metadata
|
||||
if (a === 172 && b >= 16 && b <= 31) { return true } // RFC1918
|
||||
if (a === 192 && b === 168) { return true } // RFC1918
|
||||
if (a === 192 && b === 0) { return true } // 192.0.0/24 + 192.0.2/24
|
||||
if (a === 198 && (b === 18 || b === 19)) { return true } // benchmarking
|
||||
if (a === 100 && b >= 64 && b <= 127) { return true } // CGNAT
|
||||
if (a === 198 && b === 51) { return true } // TEST-NET-2
|
||||
if (a === 203 && b === 0) { return true } // TEST-NET-3
|
||||
if (a >= 224) { return true } // multicast + reserved
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
const isPrivateIpv6 = (address) => {
|
||||
const lower = address.toLowerCase().replace(/^\[|\]$/g, '')
|
||||
|
||||
// Unwrap v4-mapped (::ffff:127.0.0.1) and NAT64 (64:ff9b::/96) before
|
||||
// judging, otherwise ::ffff:127.0.0.1 reads as "some v6 address".
|
||||
const mapped = lower.match(/^(?:::ffff:|64:ff9b::)(\d+\.\d+\.\d+\.\d+)$/)
|
||||
if (mapped) { return isPrivateIpv4(mapped[1]) }
|
||||
|
||||
if (lower === '::' || lower === '::1') { return true } // unspecified, loopback
|
||||
if (/^f[cd]/.test(lower)) { return true } // fc00::/7 unique-local
|
||||
if (/^fe[89ab]/.test(lower)) { return true } // fe80::/10 link-local
|
||||
if (/^ff/.test(lower)) { return true } // multicast
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
//
|
||||
// The gate. Throws SafetyError; resolves to the vetted addresses on success.
|
||||
//
|
||||
// Known limitation (documented, accepted): this resolves DNS and then hands the
|
||||
// hostname to fetch, which resolves again. A hostile 0-TTL record can answer
|
||||
// differently the second time. Closing that properly needs a custom undici
|
||||
// Agent with connect.lookup - deliberately deferred, see the plan's Phase 5.
|
||||
// Residual risk is blind SSRF only: no credentials are sent and no response
|
||||
// body is returned to whoever planted the URL.
|
||||
//
|
||||
UrlSafety.assertFetchable = async (raw) => {
|
||||
const url = raw instanceof URL ? raw : UrlSafety.parse(raw)
|
||||
|
||||
if (!url) {
|
||||
throw new SafetyError('bad_url', `Not a fetchable URL: ${raw}`)
|
||||
}
|
||||
if (!FETCHABLE_SCHEMES.has(url.protocol)) {
|
||||
throw new SafetyError('bad_scheme', `Refusing scheme ${url.protocol}`)
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new SafetyError('credentials_in_url', 'Refusing URL with embedded credentials')
|
||||
}
|
||||
|
||||
const hostname = url.hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase()
|
||||
|
||||
if (!hostname) {
|
||||
throw new SafetyError('bad_url', 'URL has no host')
|
||||
}
|
||||
if (hostname === 'localhost' || BLOCKED_SUFFIXES.some(suffix => hostname.endsWith(suffix))) {
|
||||
throw new SafetyError('blocked_host', `Refusing internal host ${hostname}`)
|
||||
}
|
||||
|
||||
// Literal IP in the URL - no DNS needed.
|
||||
if (net.isIP(hostname)) {
|
||||
if (UrlSafety.isPrivateIp(hostname)) {
|
||||
throw new SafetyError('private_ip', `Refusing non-routable address ${hostname}`)
|
||||
}
|
||||
return [hostname]
|
||||
}
|
||||
|
||||
let resolved = []
|
||||
try {
|
||||
resolved = await dns.lookup(hostname, { all: true, verbatim: true })
|
||||
} catch (error) {
|
||||
throw new SafetyError('dns_failed', `Could not resolve ${hostname}: ${error.code || error.message}`)
|
||||
}
|
||||
|
||||
if (!resolved.length) {
|
||||
throw new SafetyError('dns_failed', `No addresses for ${hostname}`)
|
||||
}
|
||||
|
||||
// Every answer must be public. One private address is enough to refuse -
|
||||
// otherwise a round-robin record gets us some of the time.
|
||||
for (const entry of resolved) {
|
||||
if (UrlSafety.isPrivateIp(entry.address)) {
|
||||
throw new SafetyError('private_ip', `${hostname} resolves to non-routable ${entry.address}`)
|
||||
}
|
||||
}
|
||||
|
||||
return resolved.map(entry => entry.address)
|
||||
}
|
||||
Reference in new issue
Block a user