Deploy and schema updates for testing

This commit is contained in:
Max committed 2026-11-03 11:48:06 -07:00
1 parent 8d60aae65c
commit 81b4162117
8 files changed
+389 -86

No files matched your search

+9
View File
@@ -21,6 +21,7 @@ const HttpFetch = require('@helpers/HttpFetch')
const HostQueue = require('@helpers/HostQueue')
const Robots = require('@helpers/Robots')
const Realtime = require('@helpers/Realtime')
const Schema = require('@helpers/Schema')
const SiteScrape = require('@helpers/SiteScrape')
const ScrapeCache = require('@models/ScrapeCache')
@@ -354,6 +355,10 @@ ScrapeQueue.enqueueUrl = ({ userId, noteId, attachmentId, url }) => {
const runJob = async ({ userId, noteId, attachmentId, url }) => {
if (!settings.enabled) { return '' }
// Backstop, so no future caller can reach the DB layer unguarded when the
// scrape columns are absent.
if (!Schema.isScrapeSchemaReady()) { return '' }
const pending = perUser.get(userId) || 0
if (pending >= settings.maxPendingPerUser) {
// Left at scrape_state = 0, so recoverPending() picks it up later.
@@ -426,6 +431,10 @@ ScrapeQueue.awaitUrl = async (promise, capMs) => {
ScrapeQueue.recoverPending = async () => {
if (!settings.enabled) { return { requeued: 0, retired: 0 } }
// Every query below references scrape_state, which only exists once an
// admin has run ./applyScrapeSchema.sh.
if (!Schema.isScrapeSchemaReady()) { return { requeued: 0, retired: 0 } }
// Anything left at FETCHING died with the previous process.
await db.promise().query(
'UPDATE attachment SET scrape_state = ? WHERE scrape_state = ?',