//
// Read-through cache for scraped URL metadata, keyed by a hash of the
// normalised URL and shared across users and notes.
//
// A fresh hit means zero network I/O. A stale hit means a conditional request
// (If-None-Match / If-Modified-Since), so an unchanged page costs one 304
// rather than a full page load. Failures are backed off so a dead link is not
// refetched on every note save.
//
// Stores only public page metadata. Note content never reaches this table.
//

const db = require('@config/database')
const UrlSafety = require('@helpers/UrlSafety')

let ScrapeCache = module.exports = {}

const SUCCESS_TTL_SECONDS = 30 * 24 * 60 * 60 // 30 days

// Widening backoff on repeated failure, then give up and stay failed.
const FAILURE_BACKOFF_SECONDS = [60 * 60, 6 * 60 * 60, 24 * 60 * 60]
const MAX_FAILURES = FAILURE_BACKOFF_SECONDS.length

ScrapeCache.SUCCESS_TTL_SECONDS = SUCCESS_TTL_SECONDS
ScrapeCache.MAX_FAILURES = MAX_FAILURES

const now = () => Math.round(Date.now() / 1000)

const truncate = (value, length) => {
	if (value == null) { return null }
	const text = String(value)
	return text.length > length ? text.substring(0, length) : text
}

ScrapeCache.get = async (urlOrKey) => {
	const key = /^[0-9a-f]{64}$/.test(urlOrKey) ? urlOrKey : UrlSafety.urlKey(urlOrKey)
	if (!key) { return null }

	const [rows] = await db.promise().query('SELECT * FROM scrape_cache WHERE url_hash = ? LIMIT 1', [key])
	return rows.length ? rows[0] : null
}

// Fresh = inside its TTL and successfully fetched.
ScrapeCache.isFresh = (row, at) => {
	if (!row || !row.expires_at) { return false }
	return row.expires_at > (at || now())
}

ScrapeCache.isUsable = (row) => {
	return !!(row && row.fetch_state === 'ok' && row.search_text != null)
}

// Has this URL failed enough times that we should stop trying?
ScrapeCache.isExhausted = (row) => {
	return !!(row && row.fetch_state === 'failed' && row.failure_count >= MAX_FAILURES)
}

//
// Conditional-request headers for a stale-but-successful row. Null when there
// is nothing to revalidate against.
//
ScrapeCache.conditionalFor = (row) => {
	if (!row || row.fetch_state !== 'ok') { return null }
	if (!row.etag && !row.last_modified) { return null }

	return { etag: row.etag || null, lastModified: row.last_modified || null }
}

//
// Reserve a row before fetching, so a concurrent save for the same URL finds
// it rather than starting a second fetch.
//
ScrapeCache.markPending = async (url) => {
	const key = UrlSafety.urlKey(url)
	if (!key) { return null }

	await db.promise().query(
		`INSERT INTO scrape_cache (url_hash, url, fetch_state, fetched_at)
		 VALUES (?, ?, 'pending', ?)
		 ON DUPLICATE KEY UPDATE fetch_state = IF(fetch_state = 'ok', fetch_state, 'pending')`,
		[key, truncate(url, 1000), now()]
	)

	return key
}

//
// Store a successful scrape.
//
ScrapeCache.putSuccess = async (url, result) => {
	const key = UrlSafety.urlKey(url)
	if (!key) { return null }

	const at = now()
	const fields = {
		url: truncate(url, 1000),
		canonical_url: truncate(result.canonicalUrl, 1000),
		status: result.status || 200,
		title: truncate(result.title, 512),
		site_name: truncate(result.siteName, 255),
		description: result.description || null,
		keywords: truncate(result.keywords, 512),
		search_text: result.searchText || null,
		image_url: truncate(result.imageUrl, 1000),
		file_location: truncate(result.fileLocation, 500),
		etag: truncate(result.etag, 255),
		last_modified: truncate(result.lastModified, 128),
		content_type: truncate(result.contentType, 128),
		fetch_state: 'ok',
		failure_count: 0,
		last_error: null,
		fetched_at: at,
		expires_at: at + SUCCESS_TTL_SECONDS,
	}

	const columns = Object.keys(fields)
	const values = columns.map(column => fields[column])

	await db.promise().query(
		`INSERT INTO scrape_cache (url_hash, ${columns.join(', ')})
		 VALUES (?, ${columns.map(() => '?').join(', ')})
		 ON DUPLICATE KEY UPDATE ${columns.map(column => `${column} = VALUES(${column})`).join(', ')}`,
		[key, ...values]
	)

	return key
}

//
// A 304. The page is unchanged, so only the freshness window moves.
//
// Deliberately does NOT touch attachment.last_indexed - Attachment.search
// orders by it, so revalidating would silently reshuffle the user's list.
//
ScrapeCache.touchNotModified = async (url) => {
	const key = UrlSafety.urlKey(url)
	if (!key) { return null }

	const at = now()
	await db.promise().query(
		`UPDATE scrape_cache
		 SET fetched_at = ?, expires_at = ?, fetch_state = 'ok', failure_count = 0, last_error = NULL
		 WHERE url_hash = ?`,
		[at, at + SUCCESS_TTL_SECONDS, key]
	)

	return key
}

//
// Record a failure with a widening retry window.
//
// `blocked` (robots or the safety gate) is terminal: retrying will not change
// the answer, so it is parked for the full success TTL instead of backing off.
//
ScrapeCache.recordFailure = async (url, error) => {
	const key = UrlSafety.urlKey(url)
	if (!key) { return null }

	const existing = await ScrapeCache.get(key)
	const failureCount = (existing ? existing.failure_count : 0) + 1

	const kind = (error && error.kind) || 'unknown'
	const terminal = kind === 'blocked' || kind === 'robots_disallowed'

	let ttl = null
	if (terminal) {
		ttl = SUCCESS_TTL_SECONDS
	} else if (error && error.retryAfterMs) {
		ttl = Math.round(error.retryAfterMs / 1000)
	} else {
		ttl = FAILURE_BACKOFF_SECONDS[Math.min(failureCount, MAX_FAILURES) - 1]
	}

	const at = now()
	const state = terminal ? 'blocked' : 'failed'

	await db.promise().query(
		`INSERT INTO scrape_cache
			(url_hash, url, status, fetch_state, failure_count, last_error, fetched_at, expires_at)
		 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
		 ON DUPLICATE KEY UPDATE
			status = VALUES(status),
			fetch_state = VALUES(fetch_state),
			failure_count = VALUES(failure_count),
			last_error = VALUES(last_error),
			fetched_at = VALUES(fetched_at),
			expires_at = VALUES(expires_at)`,
		[
			key,
			truncate(url, 1000),
			(error && error.status) || 0,
			state,
			failureCount,
			truncate(`${kind}: ${(error && error.message) || ''}`, 255),
			at,
			at + ttl,
		]
	)

	return key
}

//
// Project a cache row onto the shape Attachment rows want.
//
ScrapeCache.toAttachmentFields = (row) => {
	if (!row) { return null }
	return {
		text: row.search_text || '',
		fileLocation: row.file_location || null,
		url: row.canonical_url || row.url,
	}
}

ScrapeCache.purgeExpired = async (limit = 500) => {
	const [result] = await db.promise().query(
		`DELETE FROM scrape_cache
		 WHERE expires_at IS NOT NULL AND expires_at < ? AND fetch_state <> 'ok'
		 LIMIT ?`,
		[now(), limit]
	)
	return result.affectedRows || 0
}