Adding scrape files for testing
This commit is contained in:
Max committed
1 parent
7d255fe687
commit
8d60aae65c
40 files changed
+4423
No files matched your search
@@ -0,0 +1,224 @@
|
||||
//
|
||||
// Read-through cache for scraped URL metadata, keyed by a hash of the
|
||||
// normalised URL and shared across users and notes.
|
||||
//
|
||||
// A fresh hit means zero network I/O. A stale hit means a conditional request
|
||||
// (If-None-Match / If-Modified-Since), so an unchanged page costs one 304
|
||||
// rather than a full page load. Failures are backed off so a dead link is not
|
||||
// refetched on every note save.
|
||||
//
|
||||
// Stores only public page metadata. Note content never reaches this table.
|
||||
//
|
||||
|
||||
const db = require('@config/database')
|
||||
const UrlSafety = require('@helpers/UrlSafety')
|
||||
|
||||
let ScrapeCache = module.exports = {}
|
||||
|
||||
const SUCCESS_TTL_SECONDS = 30 * 24 * 60 * 60 // 30 days
|
||||
|
||||
// Widening backoff on repeated failure, then give up and stay failed.
|
||||
const FAILURE_BACKOFF_SECONDS = [60 * 60, 6 * 60 * 60, 24 * 60 * 60]
|
||||
const MAX_FAILURES = FAILURE_BACKOFF_SECONDS.length
|
||||
|
||||
ScrapeCache.SUCCESS_TTL_SECONDS = SUCCESS_TTL_SECONDS
|
||||
ScrapeCache.MAX_FAILURES = MAX_FAILURES
|
||||
|
||||
const now = () => Math.round(Date.now() / 1000)
|
||||
|
||||
const truncate = (value, length) => {
|
||||
if (value == null) { return null }
|
||||
const text = String(value)
|
||||
return text.length > length ? text.substring(0, length) : text
|
||||
}
|
||||
|
||||
ScrapeCache.get = async (urlOrKey) => {
|
||||
const key = /^[0-9a-f]{64}$/.test(urlOrKey) ? urlOrKey : UrlSafety.urlKey(urlOrKey)
|
||||
if (!key) { return null }
|
||||
|
||||
const [rows] = await db.promise().query('SELECT * FROM scrape_cache WHERE url_hash = ? LIMIT 1', [key])
|
||||
return rows.length ? rows[0] : null
|
||||
}
|
||||
|
||||
// Fresh = inside its TTL and successfully fetched.
|
||||
ScrapeCache.isFresh = (row, at) => {
|
||||
if (!row || !row.expires_at) { return false }
|
||||
return row.expires_at > (at || now())
|
||||
}
|
||||
|
||||
ScrapeCache.isUsable = (row) => {
|
||||
return !!(row && row.fetch_state === 'ok' && row.search_text != null)
|
||||
}
|
||||
|
||||
// Has this URL failed enough times that we should stop trying?
|
||||
ScrapeCache.isExhausted = (row) => {
|
||||
return !!(row && row.fetch_state === 'failed' && row.failure_count >= MAX_FAILURES)
|
||||
}
|
||||
|
||||
//
|
||||
// Conditional-request headers for a stale-but-successful row. Null when there
|
||||
// is nothing to revalidate against.
|
||||
//
|
||||
ScrapeCache.conditionalFor = (row) => {
|
||||
if (!row || row.fetch_state !== 'ok') { return null }
|
||||
if (!row.etag && !row.last_modified) { return null }
|
||||
|
||||
return { etag: row.etag || null, lastModified: row.last_modified || null }
|
||||
}
|
||||
|
||||
//
|
||||
// Reserve a row before fetching, so a concurrent save for the same URL finds
|
||||
// it rather than starting a second fetch.
|
||||
//
|
||||
ScrapeCache.markPending = async (url) => {
|
||||
const key = UrlSafety.urlKey(url)
|
||||
if (!key) { return null }
|
||||
|
||||
await db.promise().query(
|
||||
`INSERT INTO scrape_cache (url_hash, url, fetch_state, fetched_at)
|
||||
VALUES (?, ?, 'pending', ?)
|
||||
ON DUPLICATE KEY UPDATE fetch_state = IF(fetch_state = 'ok', fetch_state, 'pending')`,
|
||||
[key, truncate(url, 1000), now()]
|
||||
)
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
//
|
||||
// Store a successful scrape.
|
||||
//
|
||||
ScrapeCache.putSuccess = async (url, result) => {
|
||||
const key = UrlSafety.urlKey(url)
|
||||
if (!key) { return null }
|
||||
|
||||
const at = now()
|
||||
const fields = {
|
||||
url: truncate(url, 1000),
|
||||
canonical_url: truncate(result.canonicalUrl, 1000),
|
||||
status: result.status || 200,
|
||||
title: truncate(result.title, 512),
|
||||
site_name: truncate(result.siteName, 255),
|
||||
description: result.description || null,
|
||||
keywords: truncate(result.keywords, 512),
|
||||
search_text: result.searchText || null,
|
||||
image_url: truncate(result.imageUrl, 1000),
|
||||
file_location: truncate(result.fileLocation, 500),
|
||||
etag: truncate(result.etag, 255),
|
||||
last_modified: truncate(result.lastModified, 128),
|
||||
content_type: truncate(result.contentType, 128),
|
||||
fetch_state: 'ok',
|
||||
failure_count: 0,
|
||||
last_error: null,
|
||||
fetched_at: at,
|
||||
expires_at: at + SUCCESS_TTL_SECONDS,
|
||||
}
|
||||
|
||||
const columns = Object.keys(fields)
|
||||
const values = columns.map(column => fields[column])
|
||||
|
||||
await db.promise().query(
|
||||
`INSERT INTO scrape_cache (url_hash, ${columns.join(', ')})
|
||||
VALUES (?, ${columns.map(() => '?').join(', ')})
|
||||
ON DUPLICATE KEY UPDATE ${columns.map(column => `${column} = VALUES(${column})`).join(', ')}`,
|
||||
[key, ...values]
|
||||
)
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
//
|
||||
// A 304. The page is unchanged, so only the freshness window moves.
|
||||
//
|
||||
// Deliberately does NOT touch attachment.last_indexed - Attachment.search
|
||||
// orders by it, so revalidating would silently reshuffle the user's list.
|
||||
//
|
||||
ScrapeCache.touchNotModified = async (url) => {
|
||||
const key = UrlSafety.urlKey(url)
|
||||
if (!key) { return null }
|
||||
|
||||
const at = now()
|
||||
await db.promise().query(
|
||||
`UPDATE scrape_cache
|
||||
SET fetched_at = ?, expires_at = ?, fetch_state = 'ok', failure_count = 0, last_error = NULL
|
||||
WHERE url_hash = ?`,
|
||||
[at, at + SUCCESS_TTL_SECONDS, key]
|
||||
)
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
//
|
||||
// Record a failure with a widening retry window.
|
||||
//
|
||||
// `blocked` (robots or the safety gate) is terminal: retrying will not change
|
||||
// the answer, so it is parked for the full success TTL instead of backing off.
|
||||
//
|
||||
ScrapeCache.recordFailure = async (url, error) => {
|
||||
const key = UrlSafety.urlKey(url)
|
||||
if (!key) { return null }
|
||||
|
||||
const existing = await ScrapeCache.get(key)
|
||||
const failureCount = (existing ? existing.failure_count : 0) + 1
|
||||
|
||||
const kind = (error && error.kind) || 'unknown'
|
||||
const terminal = kind === 'blocked' || kind === 'robots_disallowed'
|
||||
|
||||
let ttl = null
|
||||
if (terminal) {
|
||||
ttl = SUCCESS_TTL_SECONDS
|
||||
} else if (error && error.retryAfterMs) {
|
||||
ttl = Math.round(error.retryAfterMs / 1000)
|
||||
} else {
|
||||
ttl = FAILURE_BACKOFF_SECONDS[Math.min(failureCount, MAX_FAILURES) - 1]
|
||||
}
|
||||
|
||||
const at = now()
|
||||
const state = terminal ? 'blocked' : 'failed'
|
||||
|
||||
await db.promise().query(
|
||||
`INSERT INTO scrape_cache
|
||||
(url_hash, url, status, fetch_state, failure_count, last_error, fetched_at, expires_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
status = VALUES(status),
|
||||
fetch_state = VALUES(fetch_state),
|
||||
failure_count = VALUES(failure_count),
|
||||
last_error = VALUES(last_error),
|
||||
fetched_at = VALUES(fetched_at),
|
||||
expires_at = VALUES(expires_at)`,
|
||||
[
|
||||
key,
|
||||
truncate(url, 1000),
|
||||
(error && error.status) || 0,
|
||||
state,
|
||||
failureCount,
|
||||
truncate(`${kind}: ${(error && error.message) || ''}`, 255),
|
||||
at,
|
||||
at + ttl,
|
||||
]
|
||||
)
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
//
|
||||
// Project a cache row onto the shape Attachment rows want.
|
||||
//
|
||||
ScrapeCache.toAttachmentFields = (row) => {
|
||||
if (!row) { return null }
|
||||
return {
|
||||
text: row.search_text || '',
|
||||
fileLocation: row.file_location || null,
|
||||
url: row.canonical_url || row.url,
|
||||
}
|
||||
}
|
||||
|
||||
ScrapeCache.purgeExpired = async (limit = 500) => {
|
||||
const [result] = await db.promise().query(
|
||||
`DELETE FROM scrape_cache
|
||||
WHERE expires_at IS NOT NULL AND expires_at < ? AND fetch_state <> 'ok'
|
||||
LIMIT ?`,
|
||||
[now(), limit]
|
||||
)
|
||||
return result.affectedRows || 0
|
||||
}
|
||||
Reference in new issue
Block a user