SolidScribe/server/tests/helpers/SiteScrape.test.js

require('module-alias/register')

const fs = require('fs')
const path = require('path')
const cheerio = require('cheerio')

const SiteScrape = require('@helpers/SiteScrape')

const FIXTURES = path.join(__dirname, '..', 'fixtures', 'html')

// Fixtures are frozen snapshots - see fixtures/html/README.md. No network here.
const load = (name) => cheerio.load(fs.readFileSync(path.join(FIXTURES, name), 'utf8'))

const PAGE_URL = 'https://example.com/blog/2026/post'

describe('getTitle', () => {

	test('prefers og:title over the title element', () => {
		expect(SiteScrape.getTitle(load('og-complete.html'))).toBe('The Real Title')
	})

	test('falls back to twitter:title', () => {
		expect(SiteScrape.getTitle(load('twitter-only.html'))).toBe('Twitter Card Title')
	})

	test('falls back to JSON-LD headline', () => {
		expect(SiteScrape.getTitle(load('jsonld-article.html'))).toBe('JSON-LD Headline')
	})

	test('falls back to the title element and collapses whitespace', () => {
		expect(SiteScrape.getTitle(load('title-only.html'))).toBe('Just A Title')
	})

	test('falls back to an h1 when there is no title element', () => {
		expect(SiteScrape.getTitle(load('h1-fallback.html'))).toBe('Heading Is All We Have')
	})

	test('returns empty string for a page with nothing', () => {
		expect(SiteScrape.getTitle(load('no-metadata.html'))).toBe('')
	})
})

describe('getDescription', () => {

	test('prefers og:description', () => {
		expect(SiteScrape.getDescription(load('og-complete.html')))
			.toBe('A proper social description of the page.')
	})

	test('falls back to twitter:description', () => {
		expect(SiteScrape.getDescription(load('twitter-only.html'))).toBe('Twitter card description.')
	})

	test('falls back to JSON-LD description', () => {
		expect(SiteScrape.getDescription(load('jsonld-article.html'))).toBe('Described in structured data.')
	})

	test('is empty when no description exists', () => {
		expect(SiteScrape.getDescription(load('title-only.html'))).toBe('')
	})
})

describe('getJsonLd', () => {

	test('reads a plain block', () => {
		expect(SiteScrape.getJsonLd(load('jsonld-article.html')).headline).toBe('JSON-LD Headline')
	})

	test('digs the content node out of an @graph', () => {
		expect(SiteScrape.getJsonLd(load('jsonld-graph.html')).headline).toBe('Graph Headline')
	})

	// Malformed JSON-LD is extremely common and must never throw.
	test('survives malformed JSON without throwing', () => {
		expect(() => SiteScrape.getJsonLd(load('jsonld-malformed.html'))).not.toThrow()
		expect(SiteScrape.getTitle(load('jsonld-malformed.html'))).toBe('Survived Bad JSON')
	})

	test('is null when absent', () => {
		expect(SiteScrape.getJsonLd(load('title-only.html'))).toBeNull()
	})
})

describe('getCanonicalUrl', () => {

	test('uses link rel=canonical', () => {
		expect(SiteScrape.getCanonicalUrl(load('og-complete.html'), PAGE_URL))
			.toBe('https://example.com/canonical-post')
	})

	test('falls back to the requested url', () => {
		expect(SiteScrape.getCanonicalUrl(load('title-only.html'), PAGE_URL)).toBe(PAGE_URL)
	})
})

describe('getDisplayImage', () => {

	test('prefers og:image', () => {
		expect(SiteScrape.getDisplayImage(load('og-complete.html'), PAGE_URL))
			.toBe('https://cdn.example.com/hero.jpg')
	})

	test('falls back to twitter:image', () => {
		expect(SiteScrape.getDisplayImage(load('twitter-only.html'), PAGE_URL))
			.toBe('https://cdn.example.com/twitter.png')
	})

	test('falls back to the JSON-LD image, taking the first of an array', () => {
		expect(SiteScrape.getDisplayImage(load('jsonld-article.html'), PAGE_URL))
			.toBe('https://cdn.example.com/ld-primary.jpg')
	})

	test('reads an object-form JSON-LD image', () => {
		expect(SiteScrape.getDisplayImage(load('jsonld-graph.html'), PAGE_URL))
			.toBe('https://cdn.example.com/graph.png')
	})

	// Resolution against the canonical base, not string concatenation.
	test('resolves a relative path against the canonical url', () => {
		expect(SiteScrape.getDisplayImage(load('relative-images.html'), PAGE_URL))
			.toBe('https://example.com/blog/images/hero.png')
	})

	test('resolves a root-relative icon', () => {
		expect(SiteScrape.getDisplayImage(load('root-relative-icon.html'), PAGE_URL))
			.toBe('https://example.com/favicon.ico')
	})

	test('resolves a protocol-relative url', () => {
		expect(SiteScrape.getDisplayImage(load('protocol-relative-image.html'), PAGE_URL))
			.toBe('https://cdn.example.com/proto.jpg')
	})

	// The old substring filter dropped all of these.
	test('accepts .webp', () => {
		expect(SiteScrape.getDisplayImage(load('webp-og-image.html'), PAGE_URL))
			.toBe('https://cdn.example.com/hero.webp')
	})

	test('accepts an extensionless og:image endpoint', () => {
		expect(SiteScrape.getDisplayImage(load('extensionless-og-image.html'), PAGE_URL))
			.toBe('https://example.com/api/og?title=Hello&theme=dark')
	})

	test('rejects svg while allowSvgThumbnails is off', () => {
		expect(SiteScrape.getDisplayImage(load('svg-only-image.html'), PAGE_URL)).toBe('')
	})

	test('picks the largest srcset candidate', () => {
		expect(SiteScrape.getDisplayImage(load('srcset.html'), PAGE_URL))
			.toBe('https://cdn.example.com/large.jpg')
	})

	test('prefers the largest inline image by declared area', () => {
		expect(SiteScrape.getDisplayImage(load('sized-images.html'), PAGE_URL))
			.toBe('https://cdn.example.com/big.jpg')
	})

	test('skips tracking pixels and spacers', () => {
		expect(SiteScrape.getDisplayImage(load('tracking-pixels.html'), PAGE_URL))
			.toBe('https://cdn.example.com/real-photo.jpg')
	})

	// Regression: the old code read element.attribs.src directly and threw.
	test('does not throw on img tags with no src, and finds a data-src', () => {
		const $ = load('imgs-missing-src.html')
		expect(() => SiteScrape.getDisplayImage($, PAGE_URL)).not.toThrow()
		expect(SiteScrape.getDisplayImage($, PAGE_URL)).toBe('https://cdn.example.com/lazy.jpg')
	})

	test('prefers a large apple-touch-icon over a 16x16 favicon', () => {
		expect(SiteScrape.getDisplayImage(load('apple-touch-icon.html'), PAGE_URL))
			.toBe('https://example.com/touch-180.png')
	})

	test('returns empty string when there is no usable image', () => {
		expect(SiteScrape.getDisplayImage(load('no-metadata.html'), PAGE_URL)).toBe('')
	})

	test('survives malformed markup', () => {
		expect(() => SiteScrape.getDisplayImage(load('malformed-unclosed.html'), PAGE_URL)).not.toThrow()
	})
})

describe('getImageCandidates', () => {

	test('returns an ordered, deduplicated list', () => {
		const candidates = SiteScrape.getImageCandidates(load('tracking-pixels.html'), PAGE_URL)
		expect(candidates).toEqual(['https://cdn.example.com/real-photo.jpg'])
	})

	test('is empty rather than null for a bare page', () => {
		expect(SiteScrape.getImageCandidates(load('no-metadata.html'), PAGE_URL)).toEqual([])
	})
})

describe('getKeywords', () => {

	test('extracts page terms and ignores nav, script and footer chrome', () => {
		const keywords = SiteScrape.getKeywords(load('keyword-content.html'))

		expect(keywords).toMatch(/^Keywords: /)
		expect(keywords).toMatch(/beekeeping|beehive|honey/)
		expect(keywords).not.toMatch(/navigation/)
		expect(keywords).not.toMatch(/tracking/)
		expect(keywords).not.toMatch(/footer/)
	})

	test('returns empty string when there is nothing to count', () => {
		expect(SiteScrape.getKeywords(load('no-metadata.html'))).toBe('')
	})

	test('does not log page content', () => {
		// The old implementation dumped up to 6000 chars of every page.
		const spy = jest.spyOn(console, 'log').mockImplementation(() => {})
		SiteScrape.getKeywords(load('keyword-content.html'))
		expect(spy).not.toHaveBeenCalled()
		spy.mockRestore()
	})
})

describe('getHostName', () => {

	test('returns the https origin', () => {
		expect(SiteScrape.getHostName('http://www.google.com/search?q=x')).toBe('https://www.google.com')
	})

	test('returns empty string for a bad url instead of throwing', () => {
		expect(() => SiteScrape.getHostName('nonsense')).not.toThrow()
		expect(SiteScrape.getHostName('nonsense')).toBe('')
	})
})

describe('buildSearchText', () => {

	test('joins the parts that exist', () => {
		const text = SiteScrape.buildSearchText({
			title: 'T', description: 'D', siteName: 'S', keywords: 'Keywords: a, b',
		})
		expect(text).toBe('T\nD\nS\nKeywords: a, b')
	})

	test('skips empty parts', () => {
		expect(SiteScrape.buildSearchText({ title: 'T', description: '', siteName: '', keywords: '' })).toBe('T')
	})

	test('caps the length so it cannot bloat attachment.text', () => {
		const text = SiteScrape.buildSearchText({ title: 'x'.repeat(50000) })
		expect(text.length).toBeLessThanOrEqual(8000)
	})

	// Sentinel strings used to land in attachment.text and pollute FULLTEXT.
	test('never produces the old failure sentinels', () => {
		const text = SiteScrape.buildSearchText({ title: '', description: '', siteName: '', keywords: '' })
		expect(text).toBe('')
		expect(text).not.toContain('No site text')
		expect(text).not.toContain('Request Timeout')
	})
})

describe('parse', () => {

	test('returns everything a link preview needs in one call', () => {
		const result = SiteScrape.parse(load('og-complete.html'), PAGE_URL)

		expect(result).toMatchObject({
			title: 'The Real Title',
			description: 'A proper social description of the page.',
			siteName: 'Example Journal',
			canonicalUrl: 'https://example.com/canonical-post',
			imageUrl: 'https://cdn.example.com/hero.jpg',
			hostname: 'https://example.com',
		})
		expect(result.searchText).toContain('The Real Title')
		expect(result.searchText).toContain('A proper social description')
	})

	test('degrades to empty fields rather than throwing on a bare page', () => {
		const result = SiteScrape.parse(load('no-metadata.html'), PAGE_URL)

		expect(result.title).toBe('')
		expect(result.imageUrl).toBe('')
		expect(result.searchText).toBe('example.com')
	})
})

//
// getCleanUrls is deliberately unchanged - it decides which URLs are detected in
// note text and what string is stored in attachment.url. These tests pin the
// CURRENT behaviour so the rewrite around it cannot drift link detection.
//
describe('getCleanUrls (pinned, must not change)', () => {

	test('finds bare and full urls', () => {
		expect(SiteScrape.getCleanUrls('see https://example.com/a and www.foo.com here'))
			.toEqual(['https://example.com/a', 'https://www.foo.com'])
	})

	test('still forces http to https', () => {
		expect(SiteScrape.getCleanUrls('http://example.com/x')).toEqual(['https://example.com/x'])
	})

	test('still prepends https to a bare www host', () => {
		expect(SiteScrape.getCleanUrls('www.example.com')).toEqual(['https://www.example.com'])
	})

	test('deduplicates', () => {
		expect(SiteScrape.getCleanUrls('https://a.com https://a.com')).toEqual(['https://a.com'])
	})

	test('returns an empty array when there are no urls', () => {
		expect(SiteScrape.getCleanUrls('just some words')).toEqual([])
	})

	test('returns an empty array for empty text', () => {
		expect(SiteScrape.getCleanUrls('')).toEqual([])
	})
})