SolidScribe/server/helpers/SiteScrape.js

//
// Pure parsing of an already-fetched page. No network I/O, no logging.
//
// $ = the cheerio scrape object
//
// Everything here is a pure function of ($, url), which is what makes it
// testable against saved HTML fixtures instead of the live internet.
//

const config = require('@config/scrape')

let SiteScrape = module.exports = {}

const removeWhitespace = /\s+/g

const commonWords = ['just','start','what','these','how', 'was', 'being','can','way','share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
	'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
	'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
	'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
	'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
	'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']

const clean = (value, maxLength) => {
	if (value == null) { return '' }
	const text = String(value).replace(removeWhitespace, ' ').trim()
	return maxLength ? text.substring(0, maxLength) : text
}

// First non-empty wins. Each candidate is a thunk so a missing branch costs
// nothing and nothing throws on the way to the answer.
const firstOf = (candidates) => {
	for (const candidate of candidates) {
		let value = ''
		try { value = clean(typeof candidate === 'function' ? candidate() : candidate) }
		catch (error) { value = '' }
		if (value) { return value }
	}
	return ''
}

const metaContent = ($, selector) => {
	const node = $(selector).first()
	return node.length ? node.attr('content') : ''
}

//
// JSON-LD. Malformed JSON-LD is extremely common, so every parse is guarded
// and a bad block is skipped rather than failing the whole page.
//
const LD_TYPES = ['Article', 'NewsArticle', 'BlogPosting', 'WebPage', 'Product', 'VideoObject', 'Recipe']

SiteScrape.getJsonLd = ($) => {
	const blocks = []

	$('script[type="application/ld+json"]').each((index, element) => {
		const raw = $(element).contents().text()
		if (!raw || !raw.trim()) { return }

		let parsed = null
		try { parsed = JSON.parse(raw) } catch (error) { return }

		// A block may be an object, an array, or wrap its payload in @graph.
		const flatten = (node) => {
			if (!node || typeof node !== 'object') { return }
			if (Array.isArray(node)) { return node.forEach(flatten) }
			if (Array.isArray(node['@graph'])) { node['@graph'].forEach(flatten) }
			blocks.push(node)
		}
		flatten(parsed)
	})

	if (!blocks.length) { return null }

	// Prefer a block whose @type looks like page content.
	const typed = blocks.find(block => {
		const type = block['@type']
		const types = Array.isArray(type) ? type : [type]
		return types.some(entry => LD_TYPES.includes(entry))
	})

	return typed || blocks[0]
}

const jsonLdImage = (block) => {
	if (!block || !block.image) { return '' }

	const image = Array.isArray(block.image) ? block.image[0] : block.image
	if (!image) { return '' }
	if (typeof image === 'string') { return image }
	if (typeof image === 'object') { return image.url || image.contentUrl || '' }

	return ''
}

//
// Page title.
//
SiteScrape.getTitle = ($) => {
	const ld = SiteScrape.getJsonLd($)

	return firstOf([
		() => metaContent($, 'meta[property="og:title"]'),
		() => metaContent($, 'meta[name="twitter:title"]'),
		() => ld && (ld.headline || ld.name),
		() => $('title').first().text(),
		() => $('h1').first().text(),
	]).substring(0, 500)
}

//
// Page description. The old parser extracted none at all.
//
SiteScrape.getDescription = ($) => {
	const ld = SiteScrape.getJsonLd($)

	return firstOf([
		() => metaContent($, 'meta[property="og:description"]'),
		() => metaContent($, 'meta[name="twitter:description"]'),
		() => metaContent($, 'meta[name="description"]'),
		() => ld && ld.description,
	]).substring(0, 1000)
}

SiteScrape.getSiteName = ($, url) => {
	return firstOf([
		() => metaContent($, 'meta[property="og:site_name"]'),
		() => metaContent($, 'meta[name="application-name"]'),
		() => {
			try { return new URL(url).hostname.replace(/^www\./, '') }
			catch (error) { return '' }
		},
	]).substring(0, 255)
}

//
// Canonical URL. Used as the base for resolving relative asset paths, which is
// what the old manual hostname-concatenation got wrong.
//
SiteScrape.getCanonicalUrl = ($, base) => {
	const candidate = firstOf([
		() => $('link[rel="canonical"]').first().attr('href'),
		() => metaContent($, 'meta[property="og:url"]'),
	])

	if (!candidate) { return base || '' }

	try { return new URL(candidate, base || undefined).href }
	catch (error) { return base || '' }
}

//Site hostname with https:// eg: https://www.google.com
SiteScrape.getHostName = (url) => {
	try {
		return 'https://' + (new URL(url)).hostname
	} catch (error) {
		return ''
	}
}

//
// Image selection.
//
// Extension allowlist plus an allow-if-extensionless rule. The old substring
// filter required .jpg/.jpeg/.png/.gif anywhere in the string, which silently
// dropped .webp, .avif, .ico and - more importantly - every extensionless
// og:image endpoint like /og?title=x, which is what most modern sites serve.
//
const IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.avif', '.ico', '.bmp', '.jfif']
const TRACKING_PIXEL = /(^|[\/_\-.])(1x1|pixel|spacer|blank|transparent|dot)\.(gif|png)/i

const acceptableImagePath = (href) => {
	let pathname = ''
	try { pathname = new URL(href).pathname.toLowerCase() }
	catch (error) { return false }

	const lastSegment = pathname.split('/').pop() || ''
	const dot = lastSegment.lastIndexOf('.')

	// No extension at all - allow it. An og:image endpoint is the common case.
	if (dot === -1) { return true }

	const extension = lastSegment.slice(dot)

	if (extension === '.svg') { return !!config.allowSvgThumbnails }

	return IMAGE_EXTENSIONS.includes(extension)
}

// Resolve against the page's canonical base. Handles protocol-relative
// (//host/x), root-relative, and path-relative uniformly - no string surgery.
const resolveImage = (src, base) => {
	if (!src) { return '' }

	const value = String(src).trim()
	if (!value || value.startsWith('data:')) { return '' }

	try { return new URL(value, base).href }
	catch (error) { return '' }
}

// Largest candidate in a srcset, by width descriptor.
const largestFromSrcset = (srcset) => {
	if (!srcset) { return '' }

	let best = ''
	let bestWidth = -1

	for (const entry of String(srcset).split(',')) {
		const parts = entry.trim().split(/\s+/)
		if (!parts[0]) { continue }

		const descriptor = parts[1] || ''
		const width = descriptor.endsWith('w') ? parseInt(descriptor, 10) : 0

		if (width > bestWidth) {
			bestWidth = width
			best = parts[0]
		}
	}

	return best
}

const iconCandidates = ($) => {
	const icons = []

	// apple-touch-icon is usually the largest and cleanest of the icons.
	$('link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"]').each((index, element) => {
		const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
		icons.push({ href: $(element).attr('href'), weight: 1000 + sizes })
	})

	$('link[rel="icon"], link[rel="shortcut icon"]').each((index, element) => {
		const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
		icons.push({ href: $(element).attr('href'), weight: sizes })
	})

	return icons
		.sort((a, b) => b.weight - a.weight)
		.map(icon => icon.href)
}

const inlineImageCandidates = ($) => {
	const images = []

	$('img').each((index, element) => {
		const node = $(element)

		// Guarded: the old code read element.attribs.src directly and threw on
		// an <img> with no attributes.
		const src = node.attr('src') || node.attr('data-src') || largestFromSrcset(node.attr('srcset'))
		if (!src) { return }

		const width = parseInt(node.attr('width') || '0', 10) || 0
		const height = parseInt(node.attr('height') || '0', 10) || 0

		// Tracking pixels and spacers.
		if ((width > 0 && width <= 1) || (height > 0 && height <= 1)) { return }
		if (TRACKING_PIXEL.test(src)) { return }

		images.push({ src, area: width * height, order: index })
	})

	// Biggest first; ties keep document order, so the top of the page wins.
	return images
		.sort((a, b) => (b.area - a.area) || (a.order - b.order))
		.map(image => image.src)
}

//
// Ordered candidate list, so "which image" is an explicit priority rather than
// the old reverse-iterate-and-overwrite loop that only worked by accident.
//
SiteScrape.getImageCandidates = ($, url) => {
	const base = SiteScrape.getCanonicalUrl($, url) || url
	const ld = SiteScrape.getJsonLd($)

	const raw = [
		metaContent($, 'meta[property="og:image:secure_url"]'),
		metaContent($, 'meta[property="og:image:url"]'),
		metaContent($, 'meta[property="og:image"]'),
		metaContent($, 'meta[name="twitter:image"]'),
		metaContent($, 'meta[name="twitter:image:src"]'),
		jsonLdImage(ld),
		...inlineImageCandidates($),
		...iconCandidates($),
	]

	const seen = new Set()
	const candidates = []

	for (const entry of raw) {
		const resolved = resolveImage(entry, base)
		if (!resolved || seen.has(resolved)) { continue }
		if (!acceptableImagePath(resolved)) { continue }

		seen.add(resolved)
		candidates.push(resolved)
	}

	return candidates
}

// URL for image that can be downloaded to represent website
SiteScrape.getDisplayImage = ($, url) => {
	const candidates = SiteScrape.getImageCandidates($, url)
	return candidates.length ? candidates[0] : ''
}

// Get all the site text and parse out the words that appear most
SiteScrape.getKeywords = ($) => {

	// Strip chrome and scripts before counting, otherwise navigation labels and
	// inline JS identifiers dominate the frequency table.
	const working = $.root().clone()
	working.find('script, style, noscript, nav, header, footer, aside, form, svg').remove()

	let source = working.find('article, main, [role=main], [class*=content]').text()
	if (!source || source.trim().length < 200) {
		source = working.find('body').text() || working.text()
	}

	const majorContent = String(source)
		.replace(removeWhitespace, ' ')
		.substring(0, config.maxKeywordSourceChars)
		.toLowerCase()
		.replace(/[^A-Za-z0-9- ]/g, '')

	//Count frequency of each word in scraped text
	let frequency = {}
	majorContent.split(' ').forEach(word => {
		// Exclude short or common words
		if(commonWords.includes(word) || word.length <= 2){
			return
		}
		if(!frequency[word]){
			frequency[word] = 0
		}
		// Skip some plurals
		if(frequency[word+'s'] || frequency[word+'es']){
			return
		}
		frequency[word]++
	})

	//Create a sortable array
	var sortable = [];
	for (var index in frequency) {
		if(frequency[index] > 1){
			sortable.push([index, frequency[index]]);
		}
	}

	//Sort them by most used words in the list
	sortable.sort(function(a, b) {
		return b[1] - a[1];
	});

	let finalWords = []
	for(let i=0; i<6; i++){
		if(sortable[i] && sortable[i][0]){
			finalWords.push(sortable[i][0])
		}
	}

	if(finalWords.length > 0){
		return 'Keywords: ' + finalWords.join(', ')
	}
	return ''
}

//
// The plaintext blob stored in attachment.text and indexed by the MySQL
// FULLTEXT index. Must stay unencrypted for MATCH ... AGAINST to work.
//
SiteScrape.buildSearchText = ({ title, description, siteName, keywords }) => {
	const parts = []

	if (title) { parts.push(title) }
	if (description) { parts.push(description) }
	if (siteName) { parts.push(siteName) }
	if (keywords) { parts.push(keywords) }

	return parts.join('\n').substring(0, config.maxSearchTextBytes)
}

//
// One call, everything a link preview needs.
//
SiteScrape.parse = ($, url) => {
	const canonicalUrl = SiteScrape.getCanonicalUrl($, url)
	const title = SiteScrape.getTitle($)
	const description = SiteScrape.getDescription($)
	const siteName = SiteScrape.getSiteName($, url)
	const keywords = SiteScrape.getKeywords($)
	const imageCandidates = SiteScrape.getImageCandidates($, url)

	return {
		title,
		description,
		siteName,
		canonicalUrl,
		keywords,
		imageUrl: imageCandidates.length ? imageCandidates[0] : '',
		imageCandidates,
		hostname: SiteScrape.getHostName(url),
		searchText: SiteScrape.buildSearchText({ title, description, siteName, keywords }),
	}
}

//
// Finds all urls in text, removes duplicates, makes sure they have https://
//
// DELIBERATELY UNCHANGED. This decides which URLs are detected in note text and
// what string lands in attachment.url, so altering it would change link
// behaviour for every existing note - explicitly out of scope for the scraper
// rewrite. The @TODO below is left in place for the same reason.
//
SiteScrape.getCleanUrls = (textBlock) => {
	//Find all URLs in text
	//@TODO - Use the process text library for this function
	const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
	let allUrls = textBlock.match(urlPattern)

	if(allUrls == null){
		return []
	}

	//Every URL needs HTTPS!!!
	let foundUrls = []
	allUrls.forEach( (item, index) => {
		//add protocol if it is missing
		if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
			allUrls[index] = 'https://'+item
		}
		//convert http to https
		if(item.indexOf('http://') >= 0){
			allUrls[index] = item.replace('http://','https://')
		}
	})

	//Remove all duplicates
	foundUrls = [...new Set(allUrls)]

	return foundUrls
}