Max/SolidScribe
1
0 Fork 0
Code Issues 24 Pull Requests Releases Wiki Activity

Scraper Update for testing

Browse Source
This commit is contained in:
Max committed 2026-11-03 10:23:03 -07:00
1 parent 7f38444707
commit 26e31f1885
8 files changed
+852 -828
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/models/Attachment.js
+250 -259
View File
@@ -6,15 +6,30 @@ const cs = require('@helpers/CryptoString')
let Attachment = module.exports = {}
const cheerio = require('cheerio')
const rp = require('request-promise')
const request = require('request')
const fs = require('fs')
const path = require('path')
const crypto = require('crypto')
const gm = require('gm')
const tesseract = require("node-tesseract-ocr")
// Left as-is deliberately: the upload / OCR / thumbnail code below resolves
// against it, and it is correct under pm2 (cwd = server/). Not touching it
// keeps the upload path byte-identical.
const filePath = '../staticFiles/'
// Absolute equivalent, used by the scrape path only. filePath is CWD-relative,
// so under Jest (cwd = repo root) scrape downloads landed in a sibling of the
// repo rather than in staticFiles/. This resolves to the same directory as
// filePath under pm2.
const scrapeFilePath = path.join(__dirname, '..', '..', 'staticFiles') + path.sep
const config = require('@config/scrape')
const UrlSafety = require('@helpers/UrlSafety')
const HttpFetch = require('@helpers/HttpFetch')
const Realtime = require('@helpers/Realtime')
// Attachment.migrateOld
Attachment.textSearch = (userId, searchTerm) => {
@@ -263,264 +278,222 @@ Attachment.generateThumbnail = (fileName) => {
})
}
//Scans text for websites, returns all attachments
Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
return new Promise((resolve, reject) => {
//
// Diff the URLs currently in a note against the attachments already stored for
// it, soft-delete the ones that vanished, and enqueue the new ones.
//
// Returns immediately with the text already known from cache; freshly scraped
// text arrives asynchronously via the worker. Nothing here waits on the network.
//
Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
let solrAttachmentText = '' //Final searchable scrape text for note
const ScrapeQueue = require('@workers/ScrapeQueue')
if(noteText.length == 0){ resolve(solrAttachmentText) }
// Fixed: the old version called resolve() without returning, so an empty
// note fell through and ran the DB lookup anyway.
if (!noteText || noteText.length == 0) { return Promise.resolve('') }
Attachment.urlForNote(userId, noteId).then(attachments => {
let knownText = ''
//Pull all the URLs out of the text
let foundUrls = SiteScrape.getCleanUrls(noteText)
return Attachment.urlForNote(userId, noteId)
.then(attachments => {
const foundUrls = SiteScrape.getCleanUrls(noteText)
const remaining = foundUrls.slice()
//Go through each saved URL, remove new URLs from saved URLs
//If a URL is not found, delete it
attachments.forEach(attachment => {
//URL already scraped, push text and continue
let urlIndex = foundUrls.indexOf( attachment.url )
if(urlIndex != -1){
solrAttachmentText += attachment.text
foundUrls.splice(urlIndex, 1) //Remove existing from set of found
// Compared on normalised keys rather than raw strings, so an
// http/https or www. variant is not treated as a different
// resource and needlessly deleted and re-scraped.
const index = remaining.findIndex(url =>
url === attachment.url || UrlSafety.sameResource(url, attachment.url))
if (index !== -1) {
knownText += attachment.text || ''
remaining.splice(index, 1)
} else {
//If existing attachment is not found in note, remove it
// Unchanged behaviour: link attachments are hidden, not deleted.
Attachment.delete(userId, attachment.id, true)
}
})
//No newly scraped URLs, resolve with looked up attachment text
if(foundUrls == null || foundUrls.length == 0){
return resolve(solrAttachmentText)
}
if (!remaining.length) { return knownText }
//Process the remaining URLs into attachments
Attachment.scrapeUrlsCreateAttachments(userId, noteId, foundUrls).then( freshlyScrapedText => {
//Once everything is done being scraped, emit new attachment events
SocketIo.to(`user:${userId}`).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(`user:${userId}`).emit('update_note_attachments')
solrAttachmentText += freshlyScrapedText
resolve(solrAttachmentText)
})
.catch(console.log)
})
})
}
//Return scraped text from each URL
Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
return new Promise((resolve, reject) => {
if(foundUrls == null || foundUrls.length == 0){
return resolve('')
}
console.log('About to scrape')
console.log(foundUrls)
let processedCount = 0
let scrapedText = ''
//Process each URL passd to function, a DB entry will be created for each scrape
foundUrls.forEach(url => {
Attachment.processUrl(userId, noteId, url).then( freshlyScrapedText => {
scrapedText += freshlyScrapedText
processedCount ++
//All URLs have been scraped, return data
if(processedCount == foundUrls.length){
console.log('All urls scraped')
return resolve(scrapedText)
}
})
.catch(error => {
console.log('Site Scrape error', error)
})
})
})
}
Attachment.downloadFileFromUrl = (url) => {
return new Promise((resolve, reject) => {
if(!url){
return resolve(null)
}
const random = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
let extension = ''
let fileName = random+'_scrape'
let thumbPath = 'thumb_'+fileName
console.log('Scraping image url', url)
console.log('Getting ready to scrape ', url)
request(url)
.on('error', error => {
console.log(error)
resolve(null)
})
.on('response', res => {
console.log(res.statusCode)
console.log(res.headers['content-type'])
//Get mime type from header content type
// extension = '.'+String(res.headers['content-type']).split('/').pop()
})
.pipe(fs.createWriteStream(filePath+thumbPath))
.on('close', () => {
//resize image if its real big
gm(filePath+thumbPath)
.resize(550) //Resize to width of 550 px
.quality(85) //compression level 0 - 100 (best)
.write(filePath+thumbPath, function (err) {
if(err){
console.log(err)
return resolve(null)
}
console.log('Saved Image')
return resolve(fileName)
// Create a shell row per new URL, then hand off to the worker.
return Promise.all(remaining.map(url =>
Attachment.createPendingUrlAttachment(userId, noteId, url)
.then(attachmentId => ({ userId, noteId, attachmentId, url }))
.catch(error => {
console.log('[scrape] could not create attachment row', url, error.message)
return null
})
))
.then(jobs => {
const valid = jobs.filter(Boolean)
})
// Fire and forget - the note save must not wait on this.
ScrapeQueue.enqueueUrls(valid)
.catch(error => console.log('[scrape] enqueue failed', error.message))
return knownText
})
})
// The old version had no catch here at all.
.catch(error => {
console.log('[scrape] scanTextForWebsites failed', error.message)
return knownText
})
}
//
// Create the shell attachment row for a URL and return its id.
//
// Returns the id BEFORE any network work begins. The old code armed a 5s
// setTimeout calling request.cancel() synchronously, before this INSERT had
// resolved - so a slow INSERT meant the timer fired against a null reference
// and threw inside a bare timer callback, taking the process down.
//
Attachment.createPendingUrlAttachment = (userId, noteId, url) => {
const created = Math.round((+new Date) / 1000)
return db.promise()
.query(
`INSERT INTO attachment
(note_id, user_id, attachment_type, text, url, last_indexed, file_location, scrape_state, enqueued_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[noteId, userId, 1, url, url, created, null, 0, created]
)
.then(([result]) => result.insertId)
}
//
// Scrape a set of URLs for a note and resolve with their combined text.
//
// Kept for the bookmarklet path. The old implementation resolved only when a
// completion counter reached foundUrls.length, so a single rejection meant the
// promise never settled and the caller hung forever. allSettled inside the
// worker removes the counter entirely.
//
Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
const ScrapeQueue = require('@workers/ScrapeQueue')
if (!foundUrls || !foundUrls.length) { return Promise.resolve('') }
return Promise.all(foundUrls.map(url =>
Attachment.createPendingUrlAttachment(userId, noteId, url)
.then(attachmentId => ({ userId, noteId, attachmentId, url }))
.catch(error => {
console.log('[scrape] could not create attachment row', url, error.message)
return null
})
))
.then(jobs => ScrapeQueue.enqueueUrls(jobs.filter(Boolean)))
.then(texts => texts.join(''))
}
//
// Download and store a preview image.
//
// The old downloadFileFromUrl had no timeout, no size limit, no User-Agent and
// no status check: it piped whatever came back straight into a write stream, so
// a 404 HTML error page was written to disk and handed to gm. It also wrote the
// file before validating anything, which is why staticFiles/ accumulated
// zero-byte orphans.
//
Attachment.downloadThumbnail = (imageUrl, pageUrl) => {
if (!imageUrl) { return Promise.resolve(null) }
// crypto.randomBytes rather than two concatenated Math.random() calls.
const fileName = crypto.randomBytes(16).toString('hex') + '_scrape'
const fullPath = scrapeFilePath + fileName
const thumbPath = scrapeFilePath + 'thumb_' + fileName
const tempPath = fullPath + '.tmp'
return HttpFetch.fetchBinary(imageUrl, {
pageUrl,
timeoutMs: config.imageTimeoutMs,
maxBytes: config.maxImageBytes,
})
.then(result => {
// Status and content-type were both gated inside fetchBinary, so by
// here we are holding bytes that really are an image.
if (!result.buffer || !result.buffer.length) { return null }
if (!magicBytesMatch(result.buffer, result.mime)) {
console.log('[scrape] image bytes do not match declared type', result.mime, imageUrl)
return null
}
// Write to a temp path, then rename, so a failure cannot leave a
// half-written file behind for the client to serve.
return fs.promises.writeFile(tempPath, result.buffer)
.then(() => fs.promises.rename(tempPath, fullPath))
.then(() => resizeToThumbnail(fullPath, thumbPath))
.then(() => fileName)
.catch(error => {
console.log('[scrape] could not store image', imageUrl, error.message)
return cleanupFiles([tempPath, fullPath, thumbPath]).then(() => null)
})
})
.catch(error => {
console.log('[scrape] image fetch failed', imageUrl, error.kind || error.message)
return null
})
}
Attachment.processUrl = (userId, noteId, url) => {
//
// Resize via gm. Degrades rather than failing: gm (and tesseract) are not in
// shell.nix buildInputs, so the binary is absent in dev - the old code returned
// null here and left the downloaded file orphaned on disk. Copying the
// full-size image to the thumbnail path keeps the preview working.
//
const resizeToThumbnail = (fullPath, thumbPath) => {
return new Promise(resolve => {
gm(fullPath)
.resize(550) //Resize to width of 550 px
.quality(85) //compression level 0 - 100 (best)
.write(thumbPath, error => {
if (!error) { return resolve(true) }
const scrapeTime = 5*1000;
return new Promise((resolve, reject) => {
const options = {
uri: url,
simple: true,
timeout: scrapeTime,
headers: {
'User-Agent':'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' //Simulate google headers
},
transform: function (body) {
return cheerio.load(body);
}
}
let requestTimeout = null
let thumbnail = null
let request = null
let created = Math.round((+new Date)/1000)
let insertedId = null
//Create a shell attachment for each URL, put in processing state
db.promise()
.query(`INSERT INTO attachment
(note_id, user_id, attachment_type, text, url, last_indexed, file_location)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
[noteId, userId, 1, url, url, created, null])
.then((rows, fields) => {
//Set two bigger variables then return request for processing
request = rp(options)
insertedId = rows[0].insertId
return request
})
.then($ => {
//Clear timeout that would end this function
clearTimeout(requestTimeout)
// let header = $('h1').text().replace(removeWhitespace, " ")
// desiredSearchText += header + "\n"
const pageTitle = SiteScrape.getTitle($)
const hostname = SiteScrape.getHostName(url)
const thumbnail = SiteScrape.getDisplayImage($, url)
const keywords = SiteScrape.getKeywords($)
var desiredSearchText = ''
desiredSearchText += pageTitle
if(keywords){
desiredSearchText += "\n " + keywords
}
console.log('Results from site scrape-------------')
console.log({
pageTitle,
hostname,
thumbnail,
keywords
console.log('[scrape] gm unavailable or failed, using full size as thumbnail')
fs.promises.copyFile(fullPath, thumbPath)
.then(() => resolve(true))
.catch(() => resolve(false))
})
// throw new Error('Ending this function early.')
// console.log('TexT Scraped')
// console.log(desiredSearchText)
created = Math.round((+new Date)/1000)
//Scrape URL for thumbnail - take filename and save in attachment
Attachment.downloadFileFromUrl(thumbnail)
.then(thumbnailFilename => {
//Update text and thumbnail filename
created = Math.round((+new Date)/1000)
db.promise()
.query(`UPDATE attachment SET
text = ?,
last_indexed = ?,
file_location = ?
WHERE id = ?
`, [desiredSearchText, created, thumbnailFilename, insertedId])
.then((rows, fields) => {
resolve(desiredSearchText) //Return found text
})
.catch(console.log)
//Create attachment in DB with scrape text and provided data
// db.promise()
// .query(`INSERT INTO attachment
// (note_id, user_id, attachment_type, text, url, last_indexed, file_location)
// VALUES (?, ?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created, thumbnailFilename])
// .then((rows, fields) => {
// resolve(desiredSearchText) //Return found text
// })
// .catch(console.log)
})
})
.catch(error => {
console.log('Scrape pooped out')
console.log('Issue with scrape', error.statusCode)
clearTimeout(requestTimeout)
return resolve('No site text')
})
requestTimeout = setTimeout( () => {
console.log('Cancel the request, its taking to long.')
request.cancel()
return resolve('Request Timeout')
}, scrapeTime )
})
}
const cleanupFiles = (paths) => {
return Promise.all(paths.map(target =>
fs.promises.unlink(target).catch(() => null)
))
}
//
// Sanity-check the bytes against the declared content type. A server can claim
// image/png and send anything.
//
const IMAGE_MAGIC = [
{ mime: 'image/png', bytes: [0x89, 0x50, 0x4e, 0x47] },
{ mime: 'image/jpeg', bytes: [0xff, 0xd8, 0xff] },
{ mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] },
{ mime: 'image/bmp', bytes: [0x42, 0x4d] },
]
const magicBytesMatch = (buffer, mime) => {
const expected = IMAGE_MAGIC.find(entry => entry.mime === mime)
// webp/avif/ico and friends have container-dependent headers - accept them
// on the content-type alone rather than guessing wrong.
if (!expected) { return true }
if (buffer.length < expected.bytes.length) { return false }
return expected.bytes.every((byte, index) => buffer[index] === byte)
}
Attachment.generatePushKey = (userId) => {
return new Promise((resolve, reject) => {
@@ -576,37 +549,55 @@ Attachment.getPushkeyBookmarklet = (userId) => {
})
}
Attachment.pushUrl = (pushkey,url) => {
return new Promise((resolve, reject) => {
//
// Bookmarklet entry point: GET /api/public/pushmebaby?pushkey=...&url=...
//
// The URL arrives straight off a query string, so it is validated here - the
// old code passed it through completely unvalidated, bypassing even the https
// normalisation that note text goes through.
//
// Response timing is preserved: the bookmarklet still answers after the scrape,
// but now with a hard cap instead of the previous unbounded hang.
//
Attachment.pushUrl = (pushkey, url) => {
let userId = null
pushkey = pushkey.replace(/ /g, '+')
const ScrapeQueue = require('@workers/ScrapeQueue')
db.promise()
let userId = null
pushkey = String(pushkey || '').replace(/ /g, '+')
return db.promise()
.query("SELECT id FROM user WHERE pushkey = ? LIMIT 1", [pushkey])
.then((rows, fields) => {
.then(([rows]) => {
if(rows[0].length == 0){
return resolve(true)
}
// Unchanged: an unknown pushkey resolves truthy and leaks nothing
// about whether the key exists.
if (rows.length == 0) { return null }
userId = rows[0][0].id
return Attachment.scrapeUrlsCreateAttachments(userId, null, [url])
userId = rows[0].id
return UrlSafety.assertFetchable(url)
.then(() => Attachment.createPendingUrlAttachment(userId, null, url))
.then(attachmentId => ScrapeQueue.awaitUrl(
ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url }),
config.totalBudgetMs
))
.catch(error => {
console.log('[scrape] pushUrl refused', url, error.reason || error.message)
return null
})
})
.then(() => {
if(typeof SocketIo != 'undefined'){
//Once everything is done being scraped, emit new attachment events
SocketIo.to(`user:${userId}`).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(`user:${userId}`).emit('update_note_attachments')
if (userId) {
Realtime.emitToUser(userId, 'update_counts')
Realtime.emitToUser(userId, 'update_note_attachments')
}
return resolve(true)
return true
})
.catch(error => {
console.log('[scrape] pushUrl failed', error.message)
return true
})
.catch(console.log)
})
}
Attachment.generateBookmarkletText = (pushKey) => {