Adding scrape files for testing

2026-11-03 11:20:03 -07:00
1 parent 7d255fe687
commit 8d60aae65c
@@ -0,0 +1,318 @@
+require('module-alias/register')
+
+const fs = require('fs')
+const path = require('path')
+const cheerio = require('cheerio')
+
+const SiteScrape = require('@helpers/SiteScrape')
+
+const FIXTURES = path.join(__dirname, '..', 'fixtures', 'html')
+
+// Fixtures are frozen snapshots - see fixtures/html/README.md. No network here.
+const load = (name) => cheerio.load(fs.readFileSync(path.join(FIXTURES, name), 'utf8'))
+
+const PAGE_URL = 'https://example.com/blog/2026/post'
+
+describe('getTitle', () => {
+
+	test('prefers og:title over the title element', () => {
+		expect(SiteScrape.getTitle(load('og-complete.html'))).toBe('The Real Title')
+	})
+
+	test('falls back to twitter:title', () => {
+		expect(SiteScrape.getTitle(load('twitter-only.html'))).toBe('Twitter Card Title')
+	})
+
+	test('falls back to JSON-LD headline', () => {
+		expect(SiteScrape.getTitle(load('jsonld-article.html'))).toBe('JSON-LD Headline')
+	})
+
+	test('falls back to the title element and collapses whitespace', () => {
+		expect(SiteScrape.getTitle(load('title-only.html'))).toBe('Just A Title')
+	})
+
+	test('falls back to an h1 when there is no title element', () => {
+		expect(SiteScrape.getTitle(load('h1-fallback.html'))).toBe('Heading Is All We Have')
+	})
+
+	test('returns empty string for a page with nothing', () => {
+		expect(SiteScrape.getTitle(load('no-metadata.html'))).toBe('')
+	})
+})
+
+describe('getDescription', () => {
+
+	test('prefers og:description', () => {
+		expect(SiteScrape.getDescription(load('og-complete.html')))
+			.toBe('A proper social description of the page.')
+	})
+
+	test('falls back to twitter:description', () => {
+		expect(SiteScrape.getDescription(load('twitter-only.html'))).toBe('Twitter card description.')
+	})
+
+	test('falls back to JSON-LD description', () => {
+		expect(SiteScrape.getDescription(load('jsonld-article.html'))).toBe('Described in structured data.')
+	})
+
+	test('is empty when no description exists', () => {
+		expect(SiteScrape.getDescription(load('title-only.html'))).toBe('')
+	})
+})
+
+describe('getJsonLd', () => {
+
+	test('reads a plain block', () => {
+		expect(SiteScrape.getJsonLd(load('jsonld-article.html')).headline).toBe('JSON-LD Headline')
+	})
+
+	test('digs the content node out of an @graph', () => {
+		expect(SiteScrape.getJsonLd(load('jsonld-graph.html')).headline).toBe('Graph Headline')
+	})
+
+	// Malformed JSON-LD is extremely common and must never throw.
+	test('survives malformed JSON without throwing', () => {
+		expect(() => SiteScrape.getJsonLd(load('jsonld-malformed.html'))).not.toThrow()
+		expect(SiteScrape.getTitle(load('jsonld-malformed.html'))).toBe('Survived Bad JSON')
+	})
+
+	test('is null when absent', () => {
+		expect(SiteScrape.getJsonLd(load('title-only.html'))).toBeNull()
+	})
+})
+
+describe('getCanonicalUrl', () => {
+
+	test('uses link rel=canonical', () => {
+		expect(SiteScrape.getCanonicalUrl(load('og-complete.html'), PAGE_URL))
+			.toBe('https://example.com/canonical-post')
+	})
+
+	test('falls back to the requested url', () => {
+		expect(SiteScrape.getCanonicalUrl(load('title-only.html'), PAGE_URL)).toBe(PAGE_URL)
+	})
+})
+
+describe('getDisplayImage', () => {
+
+	test('prefers og:image', () => {
+		expect(SiteScrape.getDisplayImage(load('og-complete.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/hero.jpg')
+	})
+
+	test('falls back to twitter:image', () => {
+		expect(SiteScrape.getDisplayImage(load('twitter-only.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/twitter.png')
+	})
+
+	test('falls back to the JSON-LD image, taking the first of an array', () => {
+		expect(SiteScrape.getDisplayImage(load('jsonld-article.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/ld-primary.jpg')
+	})
+
+	test('reads an object-form JSON-LD image', () => {
+		expect(SiteScrape.getDisplayImage(load('jsonld-graph.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/graph.png')
+	})
+
+	// Resolution against the canonical base, not string concatenation.
+	test('resolves a relative path against the canonical url', () => {
+		expect(SiteScrape.getDisplayImage(load('relative-images.html'), PAGE_URL))
+			.toBe('https://example.com/blog/images/hero.png')
+	})
+
+	test('resolves a root-relative icon', () => {
+		expect(SiteScrape.getDisplayImage(load('root-relative-icon.html'), PAGE_URL))
+			.toBe('https://example.com/favicon.ico')
+	})
+
+	test('resolves a protocol-relative url', () => {
+		expect(SiteScrape.getDisplayImage(load('protocol-relative-image.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/proto.jpg')
+	})
+
+	// The old substring filter dropped all of these.
+	test('accepts .webp', () => {
+		expect(SiteScrape.getDisplayImage(load('webp-og-image.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/hero.webp')
+	})
+
+	test('accepts an extensionless og:image endpoint', () => {
+		expect(SiteScrape.getDisplayImage(load('extensionless-og-image.html'), PAGE_URL))
+			.toBe('https://example.com/api/og?title=Hello&theme=dark')
+	})
+
+	test('rejects svg while allowSvgThumbnails is off', () => {
+		expect(SiteScrape.getDisplayImage(load('svg-only-image.html'), PAGE_URL)).toBe('')
+	})
+
+	test('picks the largest srcset candidate', () => {
+		expect(SiteScrape.getDisplayImage(load('srcset.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/large.jpg')
+	})
+
+	test('prefers the largest inline image by declared area', () => {
+		expect(SiteScrape.getDisplayImage(load('sized-images.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/big.jpg')
+	})
+
+	test('skips tracking pixels and spacers', () => {
+		expect(SiteScrape.getDisplayImage(load('tracking-pixels.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/real-photo.jpg')
+	})
+
+	// Regression: the old code read element.attribs.src directly and threw.
+	test('does not throw on img tags with no src, and finds a data-src', () => {
+		const $ = load('imgs-missing-src.html')
+		expect(() => SiteScrape.getDisplayImage($, PAGE_URL)).not.toThrow()
+		expect(SiteScrape.getDisplayImage($, PAGE_URL)).toBe('https://cdn.example.com/lazy.jpg')
+	})
+
+	test('prefers a large apple-touch-icon over a 16x16 favicon', () => {
+		expect(SiteScrape.getDisplayImage(load('apple-touch-icon.html'), PAGE_URL))
+			.toBe('https://example.com/touch-180.png')
+	})
+
+	test('returns empty string when there is no usable image', () => {
+		expect(SiteScrape.getDisplayImage(load('no-metadata.html'), PAGE_URL)).toBe('')
+	})
+
+	test('survives malformed markup', () => {
+		expect(() => SiteScrape.getDisplayImage(load('malformed-unclosed.html'), PAGE_URL)).not.toThrow()
+	})
+})
+
+describe('getImageCandidates', () => {
+
+	test('returns an ordered, deduplicated list', () => {
+		const candidates = SiteScrape.getImageCandidates(load('tracking-pixels.html'), PAGE_URL)
+		expect(candidates).toEqual(['https://cdn.example.com/real-photo.jpg'])
+	})
+
+	test('is empty rather than null for a bare page', () => {
+		expect(SiteScrape.getImageCandidates(load('no-metadata.html'), PAGE_URL)).toEqual([])
+	})
+})
+
+describe('getKeywords', () => {
+
+	test('extracts page terms and ignores nav, script and footer chrome', () => {
+		const keywords = SiteScrape.getKeywords(load('keyword-content.html'))
+
+		expect(keywords).toMatch(/^Keywords: /)
+		expect(keywords).toMatch(/beekeeping|beehive|honey/)
+		expect(keywords).not.toMatch(/navigation/)
+		expect(keywords).not.toMatch(/tracking/)
+		expect(keywords).not.toMatch(/footer/)
+	})
+
+	test('returns empty string when there is nothing to count', () => {
+		expect(SiteScrape.getKeywords(load('no-metadata.html'))).toBe('')
+	})
+
+	test('does not log page content', () => {
+		// The old implementation dumped up to 6000 chars of every page.
+		const spy = jest.spyOn(console, 'log').mockImplementation(() => {})
+		SiteScrape.getKeywords(load('keyword-content.html'))
+		expect(spy).not.toHaveBeenCalled()
+		spy.mockRestore()
+	})
+})
+
+describe('getHostName', () => {
+
+	test('returns the https origin', () => {
+		expect(SiteScrape.getHostName('http://www.google.com/search?q=x')).toBe('https://www.google.com')
+	})
+
+	test('returns empty string for a bad url instead of throwing', () => {
+		expect(() => SiteScrape.getHostName('nonsense')).not.toThrow()
+		expect(SiteScrape.getHostName('nonsense')).toBe('')
+	})
+})
+
+describe('buildSearchText', () => {
+
+	test('joins the parts that exist', () => {
+		const text = SiteScrape.buildSearchText({
+			title: 'T', description: 'D', siteName: 'S', keywords: 'Keywords: a, b',
+		})
+		expect(text).toBe('T\nD\nS\nKeywords: a, b')
+	})
+
+	test('skips empty parts', () => {
+		expect(SiteScrape.buildSearchText({ title: 'T', description: '', siteName: '', keywords: '' })).toBe('T')
+	})
+
+	test('caps the length so it cannot bloat attachment.text', () => {
+		const text = SiteScrape.buildSearchText({ title: 'x'.repeat(50000) })
+		expect(text.length).toBeLessThanOrEqual(8000)
+	})
+
+	// Sentinel strings used to land in attachment.text and pollute FULLTEXT.
+	test('never produces the old failure sentinels', () => {
+		const text = SiteScrape.buildSearchText({ title: '', description: '', siteName: '', keywords: '' })
+		expect(text).toBe('')
+		expect(text).not.toContain('No site text')
+		expect(text).not.toContain('Request Timeout')
+	})
+})
+
+describe('parse', () => {
+
+	test('returns everything a link preview needs in one call', () => {
+		const result = SiteScrape.parse(load('og-complete.html'), PAGE_URL)
+
+		expect(result).toMatchObject({
+			title: 'The Real Title',
+			description: 'A proper social description of the page.',
+			siteName: 'Example Journal',
+			canonicalUrl: 'https://example.com/canonical-post',
+			imageUrl: 'https://cdn.example.com/hero.jpg',
+			hostname: 'https://example.com',
+		})
+		expect(result.searchText).toContain('The Real Title')
+		expect(result.searchText).toContain('A proper social description')
+	})
+
+	test('degrades to empty fields rather than throwing on a bare page', () => {
+		const result = SiteScrape.parse(load('no-metadata.html'), PAGE_URL)
+
+		expect(result.title).toBe('')
+		expect(result.imageUrl).toBe('')
+		expect(result.searchText).toBe('example.com')
+	})
+})
+
+//
+// getCleanUrls is deliberately unchanged - it decides which URLs are detected in
+// note text and what string is stored in attachment.url. These tests pin the
+// CURRENT behaviour so the rewrite around it cannot drift link detection.
+//
+describe('getCleanUrls (pinned, must not change)', () => {
+
+	test('finds bare and full urls', () => {
+		expect(SiteScrape.getCleanUrls('see https://example.com/a and www.foo.com here'))
+			.toEqual(['https://example.com/a', 'https://www.foo.com'])
+	})
+
+	test('still forces http to https', () => {
+		expect(SiteScrape.getCleanUrls('http://example.com/x')).toEqual(['https://example.com/x'])
+	})
+
+	test('still prepends https to a bare www host', () => {
+		expect(SiteScrape.getCleanUrls('www.example.com')).toEqual(['https://www.example.com'])
+	})
+
+	test('deduplicates', () => {
+		expect(SiteScrape.getCleanUrls('https://a.com https://a.com')).toEqual(['https://a.com'])
+	})
+
+	test('returns an empty array when there are no urls', () => {
+		expect(SiteScrape.getCleanUrls('just some words')).toEqual([])
+	})
+
+	test('returns an empty array for empty text', () => {
+		expect(SiteScrape.getCleanUrls('')).toEqual([])
+	})
+})