SolidScribe/server/helpers/UrlSafety.js

//
// URL normalisation and the SSRF gate.
//
// Scraped URLs come from untrusted note text and from the /pushmebaby query
// param, so every URL is validated before a socket is opened - and again after
// every redirect hop, which is the hop that actually matters (a public URL that
// 302s to 127.0.0.1 is the classic bypass).
//
// Pure except for assertFetchable, which resolves DNS.
//

const crypto = require('crypto')
const dns = require('dns').promises
const net = require('net')
const config = require('@config/scrape')

let UrlSafety = module.exports = {}

const FETCHABLE_SCHEMES = new Set(['http:', 'https:'])

// Hostname suffixes that never belong to a public site.
const BLOCKED_SUFFIXES = ['.localhost', '.local', '.internal', '.home.arpa', '.lan']

// Query params that change nothing about the page but would fragment the cache.
const TRACKING_PARAMS = [
	/^utm_/i, /^fbclid$/i, /^gclid$/i, /^dclid$/i, /^msclkid$/i,
	/^mc_(e|c)id$/i, /^igshid$/i, /^ref_src$/i, /^_hs(enc|mi)$/i, /^vero_/i,
]

class SafetyError extends Error {
	constructor(reason, message) {
		super(message || reason)
		this.name = 'SafetyError'
		this.reason = reason
	}
}
UrlSafety.SafetyError = SafetyError

//
// Parse to a URL object, or null. Does NOT mutate anything that would change
// which resource is addressed - callers may store the result.
//
UrlSafety.parse = (raw, base) => {
	if (!raw || typeof raw !== 'string') { return null }

	let url = null
	try {
		url = new URL(raw.trim(), base || undefined)
	} catch (error) {
		return null
	}

	if (!FETCHABLE_SCHEMES.has(url.protocol)) { return null }
	if (url.href.length > config.maxUrlLength) { return null }

	return url
}

//
// Canonical form used ONLY for cache keys. Never store this - it deliberately
// discards tracking params and case, which would change what the user wrote.
//
UrlSafety.normalizeForKey = (raw) => {
	const url = UrlSafety.parse(raw)
	if (!url) { return null }

	url.hash = ''
	url.username = ''
	url.password = ''

	// Trailing dot on the host is legal DNS but splits the cache key.
	if (url.hostname.endsWith('.')) {
		url.hostname = url.hostname.slice(0, -1)
	}
	url.hostname = url.hostname.toLowerCase()

	// new URL already strips the default port, but be explicit.
	if ((url.protocol === 'http:' && url.port === '80') ||
		(url.protocol === 'https:' && url.port === '443')) {
		url.port = ''
	}

	for (const key of [...url.searchParams.keys()]) {
		if (TRACKING_PARAMS.some(pattern => pattern.test(key))) {
			url.searchParams.delete(key)
		}
	}
	url.searchParams.sort()

	if (url.pathname === '') { url.pathname = '/' }

	return url
}

// sha256 of the normalised href. This is scrape_cache.url_hash - char(64),
// because a UNIQUE index on varchar(1000) utf8mb4 would be 4000 bytes and
// blow InnoDB's 3072-byte key limit.
UrlSafety.urlKey = (raw) => {
	const url = UrlSafety.normalizeForKey(raw)
	if (!url) { return null }
	return crypto.createHash('sha256').update(url.href).digest('hex')
}

//
// True when two URLs address the same resource modulo scheme, "www." and a
// trailing slash.
//
// Load-bearing: scanTextForWebsites diffs the URLs currently in a note against
// the URLs already stored as attachments. Raw string equality would treat
// http://foo.com and https://foo.com as different resources and delete-then-
// rescrape the row on every save.
//
UrlSafety.sameResource = (a, b) => {
	const left = UrlSafety.normalizeForKey(a)
	const right = UrlSafety.normalizeForKey(b)
	if (!left || !right) { return false }

	const host = (url) => url.hostname.replace(/^www\./, '')
	const path = (url) => url.pathname.replace(/\/+$/, '')

	return host(left) === host(right) &&
		path(left) === path(right) &&
		left.search === right.search
}

UrlSafety.registrableHost = (hostname) => {
	return String(hostname || '').toLowerCase().replace(/^www\./, '').replace(/\.$/, '')
}

//
// Private / non-routable address detection.
//
// Covers the ranges that matter for SSRF against this app: loopback, RFC1918,
// link-local (including the 169.254.169.254 cloud metadata endpoint), CGNAT,
// and the IPv6 equivalents.
//
UrlSafety.isPrivateIp = (address) => {
	if (!address) { return true }

	const version = net.isIP(address)
	if (version === 0) { return true } // not an IP - caller resolves first

	if (version === 4) { return isPrivateIpv4(address) }
	return isPrivateIpv6(address)
}

const isPrivateIpv4 = (address) => {
	const parts = address.split('.').map(Number)
	if (parts.length !== 4 || parts.some(n => !Number.isInteger(n) || n < 0 || n > 255)) {
		return true
	}
	const [a, b] = parts

	if (a === 0) { return true }                            // 0.0.0.0/8 "this host"
	if (a === 10) { return true }                           // RFC1918
	if (a === 127) { return true }                          // loopback
	if (a === 169 && b === 254) { return true }             // link-local + metadata
	if (a === 172 && b >= 16 && b <= 31) { return true }    // RFC1918
	if (a === 192 && b === 168) { return true }             // RFC1918
	if (a === 192 && b === 0) { return true }               // 192.0.0/24 + 192.0.2/24
	if (a === 198 && (b === 18 || b === 19)) { return true } // benchmarking
	if (a === 100 && b >= 64 && b <= 127) { return true }   // CGNAT
	if (a === 198 && b === 51) { return true }              // TEST-NET-2
	if (a === 203 && b === 0) { return true }               // TEST-NET-3
	if (a >= 224) { return true }                           // multicast + reserved

	return false
}

const isPrivateIpv6 = (address) => {
	const lower = address.toLowerCase().replace(/^\[|\]$/g, '')

	// Unwrap v4-mapped (::ffff:127.0.0.1) and NAT64 (64:ff9b::/96) before
	// judging, otherwise ::ffff:127.0.0.1 reads as "some v6 address".
	const mapped = lower.match(/^(?:::ffff:|64:ff9b::)(\d+\.\d+\.\d+\.\d+)$/)
	if (mapped) { return isPrivateIpv4(mapped[1]) }

	if (lower === '::' || lower === '::1') { return true }   // unspecified, loopback
	if (/^f[cd]/.test(lower)) { return true }               // fc00::/7 unique-local
	if (/^fe[89ab]/.test(lower)) { return true }            // fe80::/10 link-local
	if (/^ff/.test(lower)) { return true }                  // multicast

	return false
}

//
// The gate. Throws SafetyError; resolves to the vetted addresses on success.
//
// Known limitation (documented, accepted): this resolves DNS and then hands the
// hostname to fetch, which resolves again. A hostile 0-TTL record can answer
// differently the second time. Closing that properly needs a custom undici
// Agent with connect.lookup - deliberately deferred, see the plan's Phase 5.
// Residual risk is blind SSRF only: no credentials are sent and no response
// body is returned to whoever planted the URL.
//
UrlSafety.assertFetchable = async (raw) => {
	const url = raw instanceof URL ? raw : UrlSafety.parse(raw)

	if (!url) {
		throw new SafetyError('bad_url', `Not a fetchable URL: ${raw}`)
	}
	if (!FETCHABLE_SCHEMES.has(url.protocol)) {
		throw new SafetyError('bad_scheme', `Refusing scheme ${url.protocol}`)
	}
	if (url.username || url.password) {
		throw new SafetyError('credentials_in_url', 'Refusing URL with embedded credentials')
	}

	const hostname = url.hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase()

	if (!hostname) {
		throw new SafetyError('bad_url', 'URL has no host')
	}
	if (hostname === 'localhost' || BLOCKED_SUFFIXES.some(suffix => hostname.endsWith(suffix))) {
		throw new SafetyError('blocked_host', `Refusing internal host ${hostname}`)
	}

	// Literal IP in the URL - no DNS needed.
	if (net.isIP(hostname)) {
		if (UrlSafety.isPrivateIp(hostname)) {
			throw new SafetyError('private_ip', `Refusing non-routable address ${hostname}`)
		}
		return [hostname]
	}

	let resolved = []
	try {
		resolved = await dns.lookup(hostname, { all: true, verbatim: true })
	} catch (error) {
		throw new SafetyError('dns_failed', `Could not resolve ${hostname}: ${error.code || error.message}`)
	}

	if (!resolved.length) {
		throw new SafetyError('dns_failed', `No addresses for ${hostname}`)
	}

	// Every answer must be public. One private address is enough to refuse -
	// otherwise a round-robin record gets us some of the time.
	for (const entry of resolved) {
		if (UrlSafety.isPrivateIp(entry.address)) {
			throw new SafetyError('private_ip', `${hostname} resolves to non-routable ${entry.address}`)
		}
	}

	return resolved.map(entry => entry.address)
}