Deploy and schema updates for testing
This commit is contained in:
1 parent
8d60aae65c
commit
81b4162117
8 files changed
+389
-86
No files matched your search
@@ -21,6 +21,7 @@ const HttpFetch = require('@helpers/HttpFetch')
|
||||
const HostQueue = require('@helpers/HostQueue')
|
||||
const Robots = require('@helpers/Robots')
|
||||
const Realtime = require('@helpers/Realtime')
|
||||
const Schema = require('@helpers/Schema')
|
||||
const SiteScrape = require('@helpers/SiteScrape')
|
||||
const ScrapeCache = require('@models/ScrapeCache')
|
||||
|
||||
@@ -354,6 +355,10 @@ ScrapeQueue.enqueueUrl = ({ userId, noteId, attachmentId, url }) => {
|
||||
const runJob = async ({ userId, noteId, attachmentId, url }) => {
|
||||
if (!settings.enabled) { return '' }
|
||||
|
||||
// Backstop, so no future caller can reach the DB layer unguarded when the
|
||||
// scrape columns are absent.
|
||||
if (!Schema.isScrapeSchemaReady()) { return '' }
|
||||
|
||||
const pending = perUser.get(userId) || 0
|
||||
if (pending >= settings.maxPendingPerUser) {
|
||||
// Left at scrape_state = 0, so recoverPending() picks it up later.
|
||||
@@ -426,6 +431,10 @@ ScrapeQueue.awaitUrl = async (promise, capMs) => {
|
||||
ScrapeQueue.recoverPending = async () => {
|
||||
if (!settings.enabled) { return { requeued: 0, retired: 0 } }
|
||||
|
||||
// Every query below references scrape_state, which only exists once an
|
||||
// admin has run ./applyScrapeSchema.sh.
|
||||
if (!Schema.isScrapeSchemaReady()) { return { requeued: 0, retired: 0 } }
|
||||
|
||||
// Anything left at FETCHING died with the previous process.
|
||||
await db.promise().query(
|
||||
'UPDATE attachment SET scrape_state = ? WHERE scrape_state = ?',
|
||||
|
||||
Reference in new issue
Block a user