Scraper Update for testing
This commit is contained in:
Max committed
1 parent
7f38444707
commit
26e31f1885
8 files changed
+852
-828
No files matched your search
+408
-181
@@ -1,26 +1,421 @@
|
||||
let SiteScrape = module.exports = {}
|
||||
|
||||
//
|
||||
// Pure parsing of an already-fetched page. No network I/O, no logging.
|
||||
//
|
||||
// $ = the cheerio scrape object
|
||||
//
|
||||
// Everything here is a pure function of ($, url), which is what makes it
|
||||
// testable against saved HTML fixtures instead of the live internet.
|
||||
//
|
||||
|
||||
const config = require('@config/scrape')
|
||||
|
||||
let SiteScrape = module.exports = {}
|
||||
|
||||
const removeWhitespace = /\s+/g
|
||||
|
||||
const commonWords = ['just','start','what','these','how', 'was', 'being','can','way','share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
|
||||
'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
|
||||
'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
|
||||
'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
|
||||
'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
|
||||
'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
|
||||
|
||||
SiteScrape.getTitle = ($) => {
|
||||
|
||||
let title = $('title').text().replace(removeWhitespace, " ")
|
||||
return title
|
||||
'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
|
||||
'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
|
||||
'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
|
||||
'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
|
||||
'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
|
||||
|
||||
const clean = (value, maxLength) => {
|
||||
if (value == null) { return '' }
|
||||
const text = String(value).replace(removeWhitespace, ' ').trim()
|
||||
return maxLength ? text.substring(0, maxLength) : text
|
||||
}
|
||||
|
||||
//Finds all urls in text, removes duplicates, makes sure they have https://
|
||||
// First non-empty wins. Each candidate is a thunk so a missing branch costs
|
||||
// nothing and nothing throws on the way to the answer.
|
||||
const firstOf = (candidates) => {
|
||||
for (const candidate of candidates) {
|
||||
let value = ''
|
||||
try { value = clean(typeof candidate === 'function' ? candidate() : candidate) }
|
||||
catch (error) { value = '' }
|
||||
if (value) { return value }
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
const metaContent = ($, selector) => {
|
||||
const node = $(selector).first()
|
||||
return node.length ? node.attr('content') : ''
|
||||
}
|
||||
|
||||
//
|
||||
// JSON-LD. Malformed JSON-LD is extremely common, so every parse is guarded
|
||||
// and a bad block is skipped rather than failing the whole page.
|
||||
//
|
||||
const LD_TYPES = ['Article', 'NewsArticle', 'BlogPosting', 'WebPage', 'Product', 'VideoObject', 'Recipe']
|
||||
|
||||
SiteScrape.getJsonLd = ($) => {
|
||||
const blocks = []
|
||||
|
||||
$('script[type="application/ld+json"]').each((index, element) => {
|
||||
const raw = $(element).contents().text()
|
||||
if (!raw || !raw.trim()) { return }
|
||||
|
||||
let parsed = null
|
||||
try { parsed = JSON.parse(raw) } catch (error) { return }
|
||||
|
||||
// A block may be an object, an array, or wrap its payload in @graph.
|
||||
const flatten = (node) => {
|
||||
if (!node || typeof node !== 'object') { return }
|
||||
if (Array.isArray(node)) { return node.forEach(flatten) }
|
||||
if (Array.isArray(node['@graph'])) { node['@graph'].forEach(flatten) }
|
||||
blocks.push(node)
|
||||
}
|
||||
flatten(parsed)
|
||||
})
|
||||
|
||||
if (!blocks.length) { return null }
|
||||
|
||||
// Prefer a block whose @type looks like page content.
|
||||
const typed = blocks.find(block => {
|
||||
const type = block['@type']
|
||||
const types = Array.isArray(type) ? type : [type]
|
||||
return types.some(entry => LD_TYPES.includes(entry))
|
||||
})
|
||||
|
||||
return typed || blocks[0]
|
||||
}
|
||||
|
||||
const jsonLdImage = (block) => {
|
||||
if (!block || !block.image) { return '' }
|
||||
|
||||
const image = Array.isArray(block.image) ? block.image[0] : block.image
|
||||
if (!image) { return '' }
|
||||
if (typeof image === 'string') { return image }
|
||||
if (typeof image === 'object') { return image.url || image.contentUrl || '' }
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
//
|
||||
// Page title.
|
||||
//
|
||||
SiteScrape.getTitle = ($) => {
|
||||
const ld = SiteScrape.getJsonLd($)
|
||||
|
||||
return firstOf([
|
||||
() => metaContent($, 'meta[property="og:title"]'),
|
||||
() => metaContent($, 'meta[name="twitter:title"]'),
|
||||
() => ld && (ld.headline || ld.name),
|
||||
() => $('title').first().text(),
|
||||
() => $('h1').first().text(),
|
||||
]).substring(0, 500)
|
||||
}
|
||||
|
||||
//
|
||||
// Page description. The old parser extracted none at all.
|
||||
//
|
||||
SiteScrape.getDescription = ($) => {
|
||||
const ld = SiteScrape.getJsonLd($)
|
||||
|
||||
return firstOf([
|
||||
() => metaContent($, 'meta[property="og:description"]'),
|
||||
() => metaContent($, 'meta[name="twitter:description"]'),
|
||||
() => metaContent($, 'meta[name="description"]'),
|
||||
() => ld && ld.description,
|
||||
]).substring(0, 1000)
|
||||
}
|
||||
|
||||
SiteScrape.getSiteName = ($, url) => {
|
||||
return firstOf([
|
||||
() => metaContent($, 'meta[property="og:site_name"]'),
|
||||
() => metaContent($, 'meta[name="application-name"]'),
|
||||
() => {
|
||||
try { return new URL(url).hostname.replace(/^www\./, '') }
|
||||
catch (error) { return '' }
|
||||
},
|
||||
]).substring(0, 255)
|
||||
}
|
||||
|
||||
//
|
||||
// Canonical URL. Used as the base for resolving relative asset paths, which is
|
||||
// what the old manual hostname-concatenation got wrong.
|
||||
//
|
||||
SiteScrape.getCanonicalUrl = ($, base) => {
|
||||
const candidate = firstOf([
|
||||
() => $('link[rel="canonical"]').first().attr('href'),
|
||||
() => metaContent($, 'meta[property="og:url"]'),
|
||||
])
|
||||
|
||||
if (!candidate) { return base || '' }
|
||||
|
||||
try { return new URL(candidate, base || undefined).href }
|
||||
catch (error) { return base || '' }
|
||||
}
|
||||
|
||||
//Site hostname with https:// eg: https://www.google.com
|
||||
SiteScrape.getHostName = (url) => {
|
||||
try {
|
||||
return 'https://' + (new URL(url)).hostname
|
||||
} catch (error) {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Image selection.
|
||||
//
|
||||
// Extension allowlist plus an allow-if-extensionless rule. The old substring
|
||||
// filter required .jpg/.jpeg/.png/.gif anywhere in the string, which silently
|
||||
// dropped .webp, .avif, .ico and - more importantly - every extensionless
|
||||
// og:image endpoint like /og?title=x, which is what most modern sites serve.
|
||||
//
|
||||
const IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.avif', '.ico', '.bmp', '.jfif']
|
||||
const TRACKING_PIXEL = /(^|[\/_\-.])(1x1|pixel|spacer|blank|transparent|dot)\.(gif|png)/i
|
||||
|
||||
const acceptableImagePath = (href) => {
|
||||
let pathname = ''
|
||||
try { pathname = new URL(href).pathname.toLowerCase() }
|
||||
catch (error) { return false }
|
||||
|
||||
const lastSegment = pathname.split('/').pop() || ''
|
||||
const dot = lastSegment.lastIndexOf('.')
|
||||
|
||||
// No extension at all - allow it. An og:image endpoint is the common case.
|
||||
if (dot === -1) { return true }
|
||||
|
||||
const extension = lastSegment.slice(dot)
|
||||
|
||||
if (extension === '.svg') { return !!config.allowSvgThumbnails }
|
||||
|
||||
return IMAGE_EXTENSIONS.includes(extension)
|
||||
}
|
||||
|
||||
// Resolve against the page's canonical base. Handles protocol-relative
|
||||
// (//host/x), root-relative, and path-relative uniformly - no string surgery.
|
||||
const resolveImage = (src, base) => {
|
||||
if (!src) { return '' }
|
||||
|
||||
const value = String(src).trim()
|
||||
if (!value || value.startsWith('data:')) { return '' }
|
||||
|
||||
try { return new URL(value, base).href }
|
||||
catch (error) { return '' }
|
||||
}
|
||||
|
||||
// Largest candidate in a srcset, by width descriptor.
|
||||
const largestFromSrcset = (srcset) => {
|
||||
if (!srcset) { return '' }
|
||||
|
||||
let best = ''
|
||||
let bestWidth = -1
|
||||
|
||||
for (const entry of String(srcset).split(',')) {
|
||||
const parts = entry.trim().split(/\s+/)
|
||||
if (!parts[0]) { continue }
|
||||
|
||||
const descriptor = parts[1] || ''
|
||||
const width = descriptor.endsWith('w') ? parseInt(descriptor, 10) : 0
|
||||
|
||||
if (width > bestWidth) {
|
||||
bestWidth = width
|
||||
best = parts[0]
|
||||
}
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
const iconCandidates = ($) => {
|
||||
const icons = []
|
||||
|
||||
// apple-touch-icon is usually the largest and cleanest of the icons.
|
||||
$('link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"]').each((index, element) => {
|
||||
const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
|
||||
icons.push({ href: $(element).attr('href'), weight: 1000 + sizes })
|
||||
})
|
||||
|
||||
$('link[rel="icon"], link[rel="shortcut icon"]').each((index, element) => {
|
||||
const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
|
||||
icons.push({ href: $(element).attr('href'), weight: sizes })
|
||||
})
|
||||
|
||||
return icons
|
||||
.sort((a, b) => b.weight - a.weight)
|
||||
.map(icon => icon.href)
|
||||
}
|
||||
|
||||
const inlineImageCandidates = ($) => {
|
||||
const images = []
|
||||
|
||||
$('img').each((index, element) => {
|
||||
const node = $(element)
|
||||
|
||||
// Guarded: the old code read element.attribs.src directly and threw on
|
||||
// an <img> with no attributes.
|
||||
const src = node.attr('src') || node.attr('data-src') || largestFromSrcset(node.attr('srcset'))
|
||||
if (!src) { return }
|
||||
|
||||
const width = parseInt(node.attr('width') || '0', 10) || 0
|
||||
const height = parseInt(node.attr('height') || '0', 10) || 0
|
||||
|
||||
// Tracking pixels and spacers.
|
||||
if ((width > 0 && width <= 1) || (height > 0 && height <= 1)) { return }
|
||||
if (TRACKING_PIXEL.test(src)) { return }
|
||||
|
||||
images.push({ src, area: width * height, order: index })
|
||||
})
|
||||
|
||||
// Biggest first; ties keep document order, so the top of the page wins.
|
||||
return images
|
||||
.sort((a, b) => (b.area - a.area) || (a.order - b.order))
|
||||
.map(image => image.src)
|
||||
}
|
||||
|
||||
//
|
||||
// Ordered candidate list, so "which image" is an explicit priority rather than
|
||||
// the old reverse-iterate-and-overwrite loop that only worked by accident.
|
||||
//
|
||||
SiteScrape.getImageCandidates = ($, url) => {
|
||||
const base = SiteScrape.getCanonicalUrl($, url) || url
|
||||
const ld = SiteScrape.getJsonLd($)
|
||||
|
||||
const raw = [
|
||||
metaContent($, 'meta[property="og:image:secure_url"]'),
|
||||
metaContent($, 'meta[property="og:image:url"]'),
|
||||
metaContent($, 'meta[property="og:image"]'),
|
||||
metaContent($, 'meta[name="twitter:image"]'),
|
||||
metaContent($, 'meta[name="twitter:image:src"]'),
|
||||
jsonLdImage(ld),
|
||||
...inlineImageCandidates($),
|
||||
...iconCandidates($),
|
||||
]
|
||||
|
||||
const seen = new Set()
|
||||
const candidates = []
|
||||
|
||||
for (const entry of raw) {
|
||||
const resolved = resolveImage(entry, base)
|
||||
if (!resolved || seen.has(resolved)) { continue }
|
||||
if (!acceptableImagePath(resolved)) { continue }
|
||||
|
||||
seen.add(resolved)
|
||||
candidates.push(resolved)
|
||||
}
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
// URL for image that can be downloaded to represent website
|
||||
SiteScrape.getDisplayImage = ($, url) => {
|
||||
const candidates = SiteScrape.getImageCandidates($, url)
|
||||
return candidates.length ? candidates[0] : ''
|
||||
}
|
||||
|
||||
// Get all the site text and parse out the words that appear most
|
||||
SiteScrape.getKeywords = ($) => {
|
||||
|
||||
// Strip chrome and scripts before counting, otherwise navigation labels and
|
||||
// inline JS identifiers dominate the frequency table.
|
||||
const working = $.root().clone()
|
||||
working.find('script, style, noscript, nav, header, footer, aside, form, svg').remove()
|
||||
|
||||
let source = working.find('article, main, [role=main], [class*=content]').text()
|
||||
if (!source || source.trim().length < 200) {
|
||||
source = working.find('body').text() || working.text()
|
||||
}
|
||||
|
||||
const majorContent = String(source)
|
||||
.replace(removeWhitespace, ' ')
|
||||
.substring(0, config.maxKeywordSourceChars)
|
||||
.toLowerCase()
|
||||
.replace(/[^A-Za-z0-9- ]/g, '')
|
||||
|
||||
//Count frequency of each word in scraped text
|
||||
let frequency = {}
|
||||
majorContent.split(' ').forEach(word => {
|
||||
// Exclude short or common words
|
||||
if(commonWords.includes(word) || word.length <= 2){
|
||||
return
|
||||
}
|
||||
if(!frequency[word]){
|
||||
frequency[word] = 0
|
||||
}
|
||||
// Skip some plurals
|
||||
if(frequency[word+'s'] || frequency[word+'es']){
|
||||
return
|
||||
}
|
||||
frequency[word]++
|
||||
})
|
||||
|
||||
//Create a sortable array
|
||||
var sortable = [];
|
||||
for (var index in frequency) {
|
||||
if(frequency[index] > 1){
|
||||
sortable.push([index, frequency[index]]);
|
||||
}
|
||||
}
|
||||
|
||||
//Sort them by most used words in the list
|
||||
sortable.sort(function(a, b) {
|
||||
return b[1] - a[1];
|
||||
});
|
||||
|
||||
let finalWords = []
|
||||
for(let i=0; i<6; i++){
|
||||
if(sortable[i] && sortable[i][0]){
|
||||
finalWords.push(sortable[i][0])
|
||||
}
|
||||
}
|
||||
|
||||
if(finalWords.length > 0){
|
||||
return 'Keywords: ' + finalWords.join(', ')
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
//
|
||||
// The plaintext blob stored in attachment.text and indexed by the MySQL
|
||||
// FULLTEXT index. Must stay unencrypted for MATCH ... AGAINST to work.
|
||||
//
|
||||
SiteScrape.buildSearchText = ({ title, description, siteName, keywords }) => {
|
||||
const parts = []
|
||||
|
||||
if (title) { parts.push(title) }
|
||||
if (description) { parts.push(description) }
|
||||
if (siteName) { parts.push(siteName) }
|
||||
if (keywords) { parts.push(keywords) }
|
||||
|
||||
return parts.join('\n').substring(0, config.maxSearchTextBytes)
|
||||
}
|
||||
|
||||
//
|
||||
// One call, everything a link preview needs.
|
||||
//
|
||||
SiteScrape.parse = ($, url) => {
|
||||
const canonicalUrl = SiteScrape.getCanonicalUrl($, url)
|
||||
const title = SiteScrape.getTitle($)
|
||||
const description = SiteScrape.getDescription($)
|
||||
const siteName = SiteScrape.getSiteName($, url)
|
||||
const keywords = SiteScrape.getKeywords($)
|
||||
const imageCandidates = SiteScrape.getImageCandidates($, url)
|
||||
|
||||
return {
|
||||
title,
|
||||
description,
|
||||
siteName,
|
||||
canonicalUrl,
|
||||
keywords,
|
||||
imageUrl: imageCandidates.length ? imageCandidates[0] : '',
|
||||
imageCandidates,
|
||||
hostname: SiteScrape.getHostName(url),
|
||||
searchText: SiteScrape.buildSearchText({ title, description, siteName, keywords }),
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Finds all urls in text, removes duplicates, makes sure they have https://
|
||||
//
|
||||
// DELIBERATELY UNCHANGED. This decides which URLs are detected in note text and
|
||||
// what string lands in attachment.url, so altering it would change link
|
||||
// behaviour for every existing note - explicitly out of scope for the scraper
|
||||
// rewrite. The @TODO below is left in place for the same reason.
|
||||
//
|
||||
SiteScrape.getCleanUrls = (textBlock) => {
|
||||
//Find all URLs in text
|
||||
//@TODO - Use the process text library for this function
|
||||
@@ -49,171 +444,3 @@ SiteScrape.getCleanUrls = (textBlock) => {
|
||||
|
||||
return foundUrls
|
||||
}
|
||||
|
||||
//Site hostname with https:// eg: https://www.google.com
|
||||
SiteScrape.getHostName = (url) => {
|
||||
|
||||
var hostname = 'https://'+(new URL(url)).hostname;
|
||||
// console.log('hostname', hostname)
|
||||
return hostname
|
||||
}
|
||||
|
||||
// URL for image that can be downloaded to represent website
|
||||
SiteScrape.getDisplayImage = ($, url) => {
|
||||
|
||||
const hostname = SiteScrape.getHostName(url)
|
||||
|
||||
let metaImg = $('[property="og:image"]')
|
||||
let shortcutIcon = $('[rel="shortcut icon"]')
|
||||
let favicon = $('[rel="icon"]')
|
||||
let randomImg = $('img')
|
||||
|
||||
//Set of images we may want gathered from various places in source
|
||||
let imagesWeWant = []
|
||||
let thumbnail = ''
|
||||
|
||||
//Scrape metadata for page image
|
||||
if(randomImg && randomImg.length > 0){
|
||||
|
||||
let imgSrcs = []
|
||||
for (let i = 0; i < randomImg.length; i++) {
|
||||
imgSrcs.push( randomImg[i].attribs.src )
|
||||
}
|
||||
|
||||
const half = Math.ceil(imgSrcs.length / 2)
|
||||
imagesWeWant = [...imgSrcs.slice(-half), ...imgSrcs.slice(0,half) ]
|
||||
|
||||
}
|
||||
//Grab the shortcut icon
|
||||
if(favicon && favicon[0] && favicon[0].attribs){
|
||||
imagesWeWant.push(favicon[0].attribs.href)
|
||||
}
|
||||
//Grab the shortcut icon
|
||||
if(shortcutIcon && shortcutIcon[0] && shortcutIcon[0].attribs){
|
||||
imagesWeWant.push(shortcutIcon[0].attribs.href)
|
||||
}
|
||||
//Grab the presentation image for the site
|
||||
if(metaImg && metaImg[0] && metaImg[0].attribs){
|
||||
imagesWeWant.unshift(metaImg[0].attribs.content)
|
||||
}
|
||||
|
||||
// console.log(imagesWeWant)
|
||||
|
||||
//Remove everything that isn't an accepted file format
|
||||
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
|
||||
|
||||
let img = String(imagesWeWant[i])
|
||||
|
||||
if(
|
||||
!img.includes('.jpg') &&
|
||||
!img.includes('.jpeg') &&
|
||||
!img.includes('.png') &&
|
||||
!img.includes('.gif')
|
||||
){
|
||||
imagesWeWant.splice(i,1)
|
||||
}
|
||||
}
|
||||
|
||||
//Find if we have absolute thumbnails or not
|
||||
let foundAbsolute = false
|
||||
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
|
||||
|
||||
let img = imagesWeWant[i]
|
||||
|
||||
//Add host name if its not included
|
||||
if(String(img).includes('//') || String(img).includes('http')){
|
||||
foundAbsolute = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
//Go through all found images. Grab the one closest to the top. Closer is better
|
||||
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
|
||||
|
||||
let img = imagesWeWant[i]
|
||||
|
||||
if(!String(img).includes('//') && foundAbsolute){
|
||||
continue;
|
||||
}
|
||||
|
||||
//Only add host to images if no absolute images were found
|
||||
if(!String(img).includes('//') ){
|
||||
if(img.indexOf('/') != 0){
|
||||
img = '/' + img
|
||||
}
|
||||
img = hostname + img
|
||||
}
|
||||
|
||||
if(img.indexOf('//') == 0){
|
||||
img = 'https:' + img //Scrape breaks without protocol
|
||||
}
|
||||
|
||||
thumbnail = img
|
||||
|
||||
}
|
||||
|
||||
return thumbnail
|
||||
}
|
||||
|
||||
// Get all the site text and parse out the words that appear most
|
||||
SiteScrape.getKeywords = ($) => {
|
||||
|
||||
let majorContent = ''
|
||||
|
||||
majorContent += $('[class*=content]').text()
|
||||
.replace(removeWhitespace, " ") //Remove all whitespace
|
||||
// .replace(/\W\s/g, '') //Remove all non alphanumeric characters
|
||||
.substring(0,6000) //Limit to 6000 characters
|
||||
.toLowerCase()
|
||||
.replace(/[^A-Za-z0-9- ]/g, '');
|
||||
|
||||
|
||||
console.log(majorContent)
|
||||
|
||||
//Count frequency of each word in scraped text
|
||||
let frequency = {}
|
||||
majorContent.split(' ').forEach(word => {
|
||||
// Exclude short or common words
|
||||
if(commonWords.includes(word) || word.length <= 2){
|
||||
return
|
||||
}
|
||||
if(!frequency[word]){
|
||||
frequency[word] = 0
|
||||
}
|
||||
// Skip some plurals
|
||||
if(frequency[word+'s'] || frequency[word+'es']){
|
||||
return
|
||||
}
|
||||
frequency[word]++
|
||||
})
|
||||
|
||||
//Create a sortable array
|
||||
var sortable = [];
|
||||
for (var index in frequency) {
|
||||
if(frequency[index] > 1){
|
||||
sortable.push([index, frequency[index]]);
|
||||
}
|
||||
}
|
||||
|
||||
//Sort them by most used words in the list
|
||||
sortable.sort(function(a, b) {
|
||||
return b[1] - a[1];
|
||||
});
|
||||
|
||||
let finalWords = []
|
||||
for(let i=0; i<6; i++){
|
||||
if(sortable[i] && sortable[i][0]){
|
||||
finalWords.push(sortable[i][0])
|
||||
}
|
||||
}
|
||||
|
||||
if(finalWords.length > 0){
|
||||
return 'Keywords: ' + finalWords.join(', ')
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
SiteScrape.getMainText = ($) => {}
|
||||
|
||||
|
||||
|
||||
+48
-4
@@ -345,10 +345,54 @@ app.use('/api/quick-note', quickNote)
|
||||
var metricTracking = require('@routes/metrictrackingController')
|
||||
app.use('/api/metric-tracking', metricTracking)
|
||||
|
||||
//Output running status
|
||||
app.listen(ports.express, () => {
|
||||
console.log(`Express: Listening on port ${ports.express}!`)
|
||||
})
|
||||
//
|
||||
// Scraper startup.
|
||||
//
|
||||
// ensureScrapeSchema() converges the scrape_cache table and the attachment
|
||||
// lifecycle columns. It is idempotent and runs before the listener binds, so
|
||||
// there is no window where requests arrive against a schema that is mid-change.
|
||||
// There is no migration framework in this project - see server/helpers/Schema.js.
|
||||
//
|
||||
const Schema = require('@helpers/Schema')
|
||||
const ScrapeQueue = require('@workers/ScrapeQueue')
|
||||
|
||||
Schema.ensureScrapeSchema()
|
||||
.then(() => {
|
||||
ScrapeQueue.start()
|
||||
|
||||
// The queue lives in memory, so a pm2 reload loses whatever was in it.
|
||||
// Every enqueued URL already has an attachment row, which makes the
|
||||
// work re-derivable without any extra table.
|
||||
return ScrapeQueue.recoverPending()
|
||||
})
|
||||
.then(recovery => {
|
||||
if (recovery && recovery.requeued) {
|
||||
console.log(`Scraper: re-queued ${recovery.requeued} pending URL(s), retired ${recovery.retired}`)
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
// A scraper that cannot start must not stop the API from serving.
|
||||
console.log('Scraper: startup failed, link previews disabled this boot:', error.message)
|
||||
})
|
||||
.finally(() => {
|
||||
//Output running status
|
||||
app.listen(ports.express, () => {
|
||||
console.log(`Express: Listening on port ${ports.express}!`)
|
||||
})
|
||||
})
|
||||
|
||||
//
|
||||
// pm2 reload sends SIGINT. Stop accepting new scrape work and let in-flight
|
||||
// requests abort via their own AbortControllers; rows left mid-fetch are
|
||||
// recovered on the next boot.
|
||||
//
|
||||
const shutdown = (signal) => {
|
||||
console.log(`Received ${signal}, stopping scrape queue`)
|
||||
try { ScrapeQueue.stop() } catch (error) { /* already down */ }
|
||||
process.exit(0)
|
||||
}
|
||||
process.on('SIGINT', () => shutdown('SIGINT'))
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'))
|
||||
|
||||
//
|
||||
//Error handlers
|
||||
|
||||
+250
-259
@@ -6,15 +6,30 @@ const cs = require('@helpers/CryptoString')
|
||||
let Attachment = module.exports = {}
|
||||
|
||||
const cheerio = require('cheerio')
|
||||
const rp = require('request-promise')
|
||||
const request = require('request')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const crypto = require('crypto')
|
||||
|
||||
const gm = require('gm')
|
||||
|
||||
const tesseract = require("node-tesseract-ocr")
|
||||
|
||||
// Left as-is deliberately: the upload / OCR / thumbnail code below resolves
|
||||
// against it, and it is correct under pm2 (cwd = server/). Not touching it
|
||||
// keeps the upload path byte-identical.
|
||||
const filePath = '../staticFiles/'
|
||||
|
||||
// Absolute equivalent, used by the scrape path only. filePath is CWD-relative,
|
||||
// so under Jest (cwd = repo root) scrape downloads landed in a sibling of the
|
||||
// repo rather than in staticFiles/. This resolves to the same directory as
|
||||
// filePath under pm2.
|
||||
const scrapeFilePath = path.join(__dirname, '..', '..', 'staticFiles') + path.sep
|
||||
|
||||
const config = require('@config/scrape')
|
||||
const UrlSafety = require('@helpers/UrlSafety')
|
||||
const HttpFetch = require('@helpers/HttpFetch')
|
||||
const Realtime = require('@helpers/Realtime')
|
||||
|
||||
// Attachment.migrateOld
|
||||
|
||||
Attachment.textSearch = (userId, searchTerm) => {
|
||||
@@ -263,264 +278,222 @@ Attachment.generateThumbnail = (fileName) => {
|
||||
})
|
||||
}
|
||||
|
||||
//Scans text for websites, returns all attachments
|
||||
Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
//
|
||||
// Diff the URLs currently in a note against the attachments already stored for
|
||||
// it, soft-delete the ones that vanished, and enqueue the new ones.
|
||||
//
|
||||
// Returns immediately with the text already known from cache; freshly scraped
|
||||
// text arrives asynchronously via the worker. Nothing here waits on the network.
|
||||
//
|
||||
Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
|
||||
|
||||
let solrAttachmentText = '' //Final searchable scrape text for note
|
||||
const ScrapeQueue = require('@workers/ScrapeQueue')
|
||||
|
||||
if(noteText.length == 0){ resolve(solrAttachmentText) }
|
||||
// Fixed: the old version called resolve() without returning, so an empty
|
||||
// note fell through and ran the DB lookup anyway.
|
||||
if (!noteText || noteText.length == 0) { return Promise.resolve('') }
|
||||
|
||||
Attachment.urlForNote(userId, noteId).then(attachments => {
|
||||
let knownText = ''
|
||||
|
||||
//Pull all the URLs out of the text
|
||||
let foundUrls = SiteScrape.getCleanUrls(noteText)
|
||||
return Attachment.urlForNote(userId, noteId)
|
||||
.then(attachments => {
|
||||
|
||||
const foundUrls = SiteScrape.getCleanUrls(noteText)
|
||||
const remaining = foundUrls.slice()
|
||||
|
||||
//Go through each saved URL, remove new URLs from saved URLs
|
||||
//If a URL is not found, delete it
|
||||
attachments.forEach(attachment => {
|
||||
//URL already scraped, push text and continue
|
||||
let urlIndex = foundUrls.indexOf( attachment.url )
|
||||
|
||||
if(urlIndex != -1){
|
||||
solrAttachmentText += attachment.text
|
||||
foundUrls.splice(urlIndex, 1) //Remove existing from set of found
|
||||
// Compared on normalised keys rather than raw strings, so an
|
||||
// http/https or www. variant is not treated as a different
|
||||
// resource and needlessly deleted and re-scraped.
|
||||
const index = remaining.findIndex(url =>
|
||||
url === attachment.url || UrlSafety.sameResource(url, attachment.url))
|
||||
|
||||
if (index !== -1) {
|
||||
knownText += attachment.text || ''
|
||||
remaining.splice(index, 1)
|
||||
} else {
|
||||
//If existing attachment is not found in note, remove it
|
||||
// Unchanged behaviour: link attachments are hidden, not deleted.
|
||||
Attachment.delete(userId, attachment.id, true)
|
||||
}
|
||||
})
|
||||
|
||||
//No newly scraped URLs, resolve with looked up attachment text
|
||||
if(foundUrls == null || foundUrls.length == 0){
|
||||
return resolve(solrAttachmentText)
|
||||
}
|
||||
if (!remaining.length) { return knownText }
|
||||
|
||||
//Process the remaining URLs into attachments
|
||||
Attachment.scrapeUrlsCreateAttachments(userId, noteId, foundUrls).then( freshlyScrapedText => {
|
||||
|
||||
//Once everything is done being scraped, emit new attachment events
|
||||
SocketIo.to(`user:${userId}`).emit('update_counts')
|
||||
|
||||
// Tell user to update attachments with scraped text
|
||||
SocketIo.to(`user:${userId}`).emit('update_note_attachments')
|
||||
|
||||
solrAttachmentText += freshlyScrapedText
|
||||
resolve(solrAttachmentText)
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
//Return scraped text from each URL
|
||||
Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
if(foundUrls == null || foundUrls.length == 0){
|
||||
return resolve('')
|
||||
}
|
||||
|
||||
console.log('About to scrape')
|
||||
console.log(foundUrls)
|
||||
|
||||
let processedCount = 0
|
||||
let scrapedText = ''
|
||||
|
||||
//Process each URL passd to function, a DB entry will be created for each scrape
|
||||
foundUrls.forEach(url => {
|
||||
Attachment.processUrl(userId, noteId, url).then( freshlyScrapedText => {
|
||||
|
||||
scrapedText += freshlyScrapedText
|
||||
processedCount ++
|
||||
|
||||
//All URLs have been scraped, return data
|
||||
if(processedCount == foundUrls.length){
|
||||
console.log('All urls scraped')
|
||||
return resolve(scrapedText)
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
console.log('Site Scrape error', error)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
Attachment.downloadFileFromUrl = (url) => {
|
||||
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
if(!url){
|
||||
return resolve(null)
|
||||
}
|
||||
|
||||
const random = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
|
||||
let extension = ''
|
||||
let fileName = random+'_scrape'
|
||||
let thumbPath = 'thumb_'+fileName
|
||||
|
||||
console.log('Scraping image url', url)
|
||||
|
||||
console.log('Getting ready to scrape ', url)
|
||||
|
||||
request(url)
|
||||
.on('error', error => {
|
||||
console.log(error)
|
||||
resolve(null)
|
||||
})
|
||||
.on('response', res => {
|
||||
console.log(res.statusCode)
|
||||
console.log(res.headers['content-type'])
|
||||
//Get mime type from header content type
|
||||
// extension = '.'+String(res.headers['content-type']).split('/').pop()
|
||||
})
|
||||
.pipe(fs.createWriteStream(filePath+thumbPath))
|
||||
.on('close', () => {
|
||||
|
||||
//resize image if its real big
|
||||
gm(filePath+thumbPath)
|
||||
.resize(550) //Resize to width of 550 px
|
||||
.quality(85) //compression level 0 - 100 (best)
|
||||
.write(filePath+thumbPath, function (err) {
|
||||
if(err){
|
||||
console.log(err)
|
||||
return resolve(null)
|
||||
}
|
||||
|
||||
console.log('Saved Image')
|
||||
return resolve(fileName)
|
||||
// Create a shell row per new URL, then hand off to the worker.
|
||||
return Promise.all(remaining.map(url =>
|
||||
Attachment.createPendingUrlAttachment(userId, noteId, url)
|
||||
.then(attachmentId => ({ userId, noteId, attachmentId, url }))
|
||||
.catch(error => {
|
||||
console.log('[scrape] could not create attachment row', url, error.message)
|
||||
return null
|
||||
})
|
||||
))
|
||||
.then(jobs => {
|
||||
const valid = jobs.filter(Boolean)
|
||||
|
||||
})
|
||||
// Fire and forget - the note save must not wait on this.
|
||||
ScrapeQueue.enqueueUrls(valid)
|
||||
.catch(error => console.log('[scrape] enqueue failed', error.message))
|
||||
|
||||
return knownText
|
||||
})
|
||||
})
|
||||
// The old version had no catch here at all.
|
||||
.catch(error => {
|
||||
console.log('[scrape] scanTextForWebsites failed', error.message)
|
||||
return knownText
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
// Create the shell attachment row for a URL and return its id.
|
||||
//
|
||||
// Returns the id BEFORE any network work begins. The old code armed a 5s
|
||||
// setTimeout calling request.cancel() synchronously, before this INSERT had
|
||||
// resolved - so a slow INSERT meant the timer fired against a null reference
|
||||
// and threw inside a bare timer callback, taking the process down.
|
||||
//
|
||||
Attachment.createPendingUrlAttachment = (userId, noteId, url) => {
|
||||
const created = Math.round((+new Date) / 1000)
|
||||
|
||||
return db.promise()
|
||||
.query(
|
||||
`INSERT INTO attachment
|
||||
(note_id, user_id, attachment_type, text, url, last_indexed, file_location, scrape_state, enqueued_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[noteId, userId, 1, url, url, created, null, 0, created]
|
||||
)
|
||||
.then(([result]) => result.insertId)
|
||||
}
|
||||
|
||||
//
|
||||
// Scrape a set of URLs for a note and resolve with their combined text.
|
||||
//
|
||||
// Kept for the bookmarklet path. The old implementation resolved only when a
|
||||
// completion counter reached foundUrls.length, so a single rejection meant the
|
||||
// promise never settled and the caller hung forever. allSettled inside the
|
||||
// worker removes the counter entirely.
|
||||
//
|
||||
Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
|
||||
|
||||
const ScrapeQueue = require('@workers/ScrapeQueue')
|
||||
|
||||
if (!foundUrls || !foundUrls.length) { return Promise.resolve('') }
|
||||
|
||||
return Promise.all(foundUrls.map(url =>
|
||||
Attachment.createPendingUrlAttachment(userId, noteId, url)
|
||||
.then(attachmentId => ({ userId, noteId, attachmentId, url }))
|
||||
.catch(error => {
|
||||
console.log('[scrape] could not create attachment row', url, error.message)
|
||||
return null
|
||||
})
|
||||
))
|
||||
.then(jobs => ScrapeQueue.enqueueUrls(jobs.filter(Boolean)))
|
||||
.then(texts => texts.join(''))
|
||||
}
|
||||
|
||||
//
|
||||
// Download and store a preview image.
|
||||
//
|
||||
// The old downloadFileFromUrl had no timeout, no size limit, no User-Agent and
|
||||
// no status check: it piped whatever came back straight into a write stream, so
|
||||
// a 404 HTML error page was written to disk and handed to gm. It also wrote the
|
||||
// file before validating anything, which is why staticFiles/ accumulated
|
||||
// zero-byte orphans.
|
||||
//
|
||||
Attachment.downloadThumbnail = (imageUrl, pageUrl) => {
|
||||
|
||||
if (!imageUrl) { return Promise.resolve(null) }
|
||||
|
||||
// crypto.randomBytes rather than two concatenated Math.random() calls.
|
||||
const fileName = crypto.randomBytes(16).toString('hex') + '_scrape'
|
||||
const fullPath = scrapeFilePath + fileName
|
||||
const thumbPath = scrapeFilePath + 'thumb_' + fileName
|
||||
const tempPath = fullPath + '.tmp'
|
||||
|
||||
return HttpFetch.fetchBinary(imageUrl, {
|
||||
pageUrl,
|
||||
timeoutMs: config.imageTimeoutMs,
|
||||
maxBytes: config.maxImageBytes,
|
||||
})
|
||||
.then(result => {
|
||||
|
||||
// Status and content-type were both gated inside fetchBinary, so by
|
||||
// here we are holding bytes that really are an image.
|
||||
if (!result.buffer || !result.buffer.length) { return null }
|
||||
if (!magicBytesMatch(result.buffer, result.mime)) {
|
||||
console.log('[scrape] image bytes do not match declared type', result.mime, imageUrl)
|
||||
return null
|
||||
}
|
||||
|
||||
// Write to a temp path, then rename, so a failure cannot leave a
|
||||
// half-written file behind for the client to serve.
|
||||
return fs.promises.writeFile(tempPath, result.buffer)
|
||||
.then(() => fs.promises.rename(tempPath, fullPath))
|
||||
.then(() => resizeToThumbnail(fullPath, thumbPath))
|
||||
.then(() => fileName)
|
||||
.catch(error => {
|
||||
console.log('[scrape] could not store image', imageUrl, error.message)
|
||||
return cleanupFiles([tempPath, fullPath, thumbPath]).then(() => null)
|
||||
})
|
||||
})
|
||||
.catch(error => {
|
||||
console.log('[scrape] image fetch failed', imageUrl, error.kind || error.message)
|
||||
return null
|
||||
})
|
||||
}
|
||||
|
||||
Attachment.processUrl = (userId, noteId, url) => {
|
||||
//
|
||||
// Resize via gm. Degrades rather than failing: gm (and tesseract) are not in
|
||||
// shell.nix buildInputs, so the binary is absent in dev - the old code returned
|
||||
// null here and left the downloaded file orphaned on disk. Copying the
|
||||
// full-size image to the thumbnail path keeps the preview working.
|
||||
//
|
||||
const resizeToThumbnail = (fullPath, thumbPath) => {
|
||||
return new Promise(resolve => {
|
||||
gm(fullPath)
|
||||
.resize(550) //Resize to width of 550 px
|
||||
.quality(85) //compression level 0 - 100 (best)
|
||||
.write(thumbPath, error => {
|
||||
if (!error) { return resolve(true) }
|
||||
|
||||
const scrapeTime = 5*1000;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
|
||||
const options = {
|
||||
uri: url,
|
||||
simple: true,
|
||||
timeout: scrapeTime,
|
||||
headers: {
|
||||
'User-Agent':'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' //Simulate google headers
|
||||
},
|
||||
transform: function (body) {
|
||||
return cheerio.load(body);
|
||||
}
|
||||
}
|
||||
|
||||
let requestTimeout = null
|
||||
let thumbnail = null
|
||||
let request = null
|
||||
let created = Math.round((+new Date)/1000)
|
||||
let insertedId = null
|
||||
|
||||
//Create a shell attachment for each URL, put in processing state
|
||||
db.promise()
|
||||
.query(`INSERT INTO attachment
|
||||
(note_id, user_id, attachment_type, text, url, last_indexed, file_location)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
[noteId, userId, 1, url, url, created, null])
|
||||
.then((rows, fields) => {
|
||||
//Set two bigger variables then return request for processing
|
||||
request = rp(options)
|
||||
insertedId = rows[0].insertId
|
||||
|
||||
return request
|
||||
})
|
||||
.then($ => {
|
||||
|
||||
//Clear timeout that would end this function
|
||||
clearTimeout(requestTimeout)
|
||||
|
||||
// let header = $('h1').text().replace(removeWhitespace, " ")
|
||||
// desiredSearchText += header + "\n"
|
||||
|
||||
const pageTitle = SiteScrape.getTitle($)
|
||||
|
||||
const hostname = SiteScrape.getHostName(url)
|
||||
|
||||
const thumbnail = SiteScrape.getDisplayImage($, url)
|
||||
|
||||
const keywords = SiteScrape.getKeywords($)
|
||||
|
||||
var desiredSearchText = ''
|
||||
desiredSearchText += pageTitle
|
||||
if(keywords){
|
||||
desiredSearchText += "\n " + keywords
|
||||
}
|
||||
|
||||
console.log('Results from site scrape-------------')
|
||||
console.log({
|
||||
pageTitle,
|
||||
hostname,
|
||||
thumbnail,
|
||||
keywords
|
||||
console.log('[scrape] gm unavailable or failed, using full size as thumbnail')
|
||||
fs.promises.copyFile(fullPath, thumbPath)
|
||||
.then(() => resolve(true))
|
||||
.catch(() => resolve(false))
|
||||
})
|
||||
|
||||
|
||||
// throw new Error('Ending this function early.')
|
||||
|
||||
|
||||
// console.log('TexT Scraped')
|
||||
// console.log(desiredSearchText)
|
||||
|
||||
created = Math.round((+new Date)/1000)
|
||||
|
||||
//Scrape URL for thumbnail - take filename and save in attachment
|
||||
Attachment.downloadFileFromUrl(thumbnail)
|
||||
.then(thumbnailFilename => {
|
||||
|
||||
//Update text and thumbnail filename
|
||||
created = Math.round((+new Date)/1000)
|
||||
db.promise()
|
||||
.query(`UPDATE attachment SET
|
||||
text = ?,
|
||||
last_indexed = ?,
|
||||
file_location = ?
|
||||
WHERE id = ?
|
||||
`, [desiredSearchText, created, thumbnailFilename, insertedId])
|
||||
.then((rows, fields) => {
|
||||
resolve(desiredSearchText) //Return found text
|
||||
})
|
||||
.catch(console.log)
|
||||
|
||||
|
||||
//Create attachment in DB with scrape text and provided data
|
||||
// db.promise()
|
||||
// .query(`INSERT INTO attachment
|
||||
// (note_id, user_id, attachment_type, text, url, last_indexed, file_location)
|
||||
// VALUES (?, ?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created, thumbnailFilename])
|
||||
// .then((rows, fields) => {
|
||||
|
||||
// resolve(desiredSearchText) //Return found text
|
||||
// })
|
||||
// .catch(console.log)
|
||||
})
|
||||
|
||||
})
|
||||
.catch(error => {
|
||||
console.log('Scrape pooped out')
|
||||
console.log('Issue with scrape', error.statusCode)
|
||||
clearTimeout(requestTimeout)
|
||||
return resolve('No site text')
|
||||
})
|
||||
|
||||
requestTimeout = setTimeout( () => {
|
||||
console.log('Cancel the request, its taking to long.')
|
||||
request.cancel()
|
||||
return resolve('Request Timeout')
|
||||
}, scrapeTime )
|
||||
})
|
||||
}
|
||||
|
||||
const cleanupFiles = (paths) => {
|
||||
return Promise.all(paths.map(target =>
|
||||
fs.promises.unlink(target).catch(() => null)
|
||||
))
|
||||
}
|
||||
|
||||
//
|
||||
// Sanity-check the bytes against the declared content type. A server can claim
|
||||
// image/png and send anything.
|
||||
//
|
||||
const IMAGE_MAGIC = [
|
||||
{ mime: 'image/png', bytes: [0x89, 0x50, 0x4e, 0x47] },
|
||||
{ mime: 'image/jpeg', bytes: [0xff, 0xd8, 0xff] },
|
||||
{ mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] },
|
||||
{ mime: 'image/bmp', bytes: [0x42, 0x4d] },
|
||||
]
|
||||
|
||||
const magicBytesMatch = (buffer, mime) => {
|
||||
const expected = IMAGE_MAGIC.find(entry => entry.mime === mime)
|
||||
|
||||
// webp/avif/ico and friends have container-dependent headers - accept them
|
||||
// on the content-type alone rather than guessing wrong.
|
||||
if (!expected) { return true }
|
||||
if (buffer.length < expected.bytes.length) { return false }
|
||||
|
||||
return expected.bytes.every((byte, index) => buffer[index] === byte)
|
||||
}
|
||||
|
||||
Attachment.generatePushKey = (userId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
@@ -576,37 +549,55 @@ Attachment.getPushkeyBookmarklet = (userId) => {
|
||||
})
|
||||
}
|
||||
|
||||
Attachment.pushUrl = (pushkey,url) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
//
|
||||
// Bookmarklet entry point: GET /api/public/pushmebaby?pushkey=...&url=...
|
||||
//
|
||||
// The URL arrives straight off a query string, so it is validated here - the
|
||||
// old code passed it through completely unvalidated, bypassing even the https
|
||||
// normalisation that note text goes through.
|
||||
//
|
||||
// Response timing is preserved: the bookmarklet still answers after the scrape,
|
||||
// but now with a hard cap instead of the previous unbounded hang.
|
||||
//
|
||||
Attachment.pushUrl = (pushkey, url) => {
|
||||
|
||||
let userId = null
|
||||
pushkey = pushkey.replace(/ /g, '+')
|
||||
const ScrapeQueue = require('@workers/ScrapeQueue')
|
||||
|
||||
db.promise()
|
||||
let userId = null
|
||||
pushkey = String(pushkey || '').replace(/ /g, '+')
|
||||
|
||||
return db.promise()
|
||||
.query("SELECT id FROM user WHERE pushkey = ? LIMIT 1", [pushkey])
|
||||
.then((rows, fields) => {
|
||||
.then(([rows]) => {
|
||||
|
||||
if(rows[0].length == 0){
|
||||
return resolve(true)
|
||||
}
|
||||
// Unchanged: an unknown pushkey resolves truthy and leaks nothing
|
||||
// about whether the key exists.
|
||||
if (rows.length == 0) { return null }
|
||||
|
||||
userId = rows[0][0].id
|
||||
return Attachment.scrapeUrlsCreateAttachments(userId, null, [url])
|
||||
userId = rows[0].id
|
||||
|
||||
return UrlSafety.assertFetchable(url)
|
||||
.then(() => Attachment.createPendingUrlAttachment(userId, null, url))
|
||||
.then(attachmentId => ScrapeQueue.awaitUrl(
|
||||
ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url }),
|
||||
config.totalBudgetMs
|
||||
))
|
||||
.catch(error => {
|
||||
console.log('[scrape] pushUrl refused', url, error.reason || error.message)
|
||||
return null
|
||||
})
|
||||
})
|
||||
.then(() => {
|
||||
|
||||
if(typeof SocketIo != 'undefined'){
|
||||
//Once everything is done being scraped, emit new attachment events
|
||||
SocketIo.to(`user:${userId}`).emit('update_counts')
|
||||
|
||||
// Tell user to update attachments with scraped text
|
||||
SocketIo.to(`user:${userId}`).emit('update_note_attachments')
|
||||
if (userId) {
|
||||
Realtime.emitToUser(userId, 'update_counts')
|
||||
Realtime.emitToUser(userId, 'update_note_attachments')
|
||||
}
|
||||
|
||||
return resolve(true)
|
||||
return true
|
||||
})
|
||||
.catch(error => {
|
||||
console.log('[scrape] pushUrl failed', error.message)
|
||||
return true
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
}
|
||||
|
||||
Attachment.generateBookmarkletText = (pushKey) => {
|
||||
|
||||
@@ -4,6 +4,7 @@ let Note = module.exports = {}
|
||||
|
||||
let Tags = require('@models/Tag')
|
||||
let Attachment = require('@models/Attachment')
|
||||
let Realtime = require('@helpers/Realtime')
|
||||
let ShareNote = require('@models/ShareNote')
|
||||
|
||||
let ProcessText = require('@helpers/ProcessText')
|
||||
@@ -12,7 +13,6 @@ const DiffMatchPatch = require('@helpers/DiffMatchPatch')
|
||||
|
||||
const crypto = require('crypto')
|
||||
const cs = require('@helpers/CryptoString')
|
||||
const rp = require('request-promise');
|
||||
const fs = require('fs')
|
||||
const gm = require('gm')
|
||||
|
||||
@@ -513,12 +513,15 @@ Note.update = (userId, noteId, noteText, noteTitle, color, pinned, archived, has
|
||||
})
|
||||
.then((rows, fields) => {
|
||||
|
||||
if(typeof SocketIo != 'undefined'){
|
||||
SocketIo.to(`user:${userId}`).emit('new_note_text_saved', {noteId, hash})
|
||||
// The old `if(typeof SocketIo != 'undefined')` guard did not just
|
||||
// skip the emit - it skipped the scrape too, so link scraping was
|
||||
// silently disabled under Jest and could never be tested. Realtime
|
||||
// no-ops safely without a socket server, so both now always run.
|
||||
Realtime.emitToUser(userId, 'new_note_text_saved', {noteId, hash})
|
||||
|
||||
//Async attachment reindex
|
||||
Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
|
||||
}
|
||||
//Async attachment reindex
|
||||
Attachment.scanTextForWebsites(userId, noteId, noteText)
|
||||
.catch(error => console.log('scanTextForWebsites failed', error.message))
|
||||
|
||||
//Send back updated response
|
||||
resolve(rows[0])
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
const Attachment = require('../../models/Attachment')
|
||||
const User = require('../../models/User')
|
||||
const ScrapeQueue = require('@workers/ScrapeQueue')
|
||||
const Robots = require('@helpers/Robots')
|
||||
const FetchMock = require('../support/fetchMock')
|
||||
const dns = require('dns').promises
|
||||
|
||||
let dnsSpy = null
|
||||
|
||||
const testUserName = 'jestTestUserAttachment'
|
||||
const password = 'Beans19934!!!'
|
||||
@@ -67,19 +73,122 @@ test('Test get Push Key Bookmarklet', () => {
|
||||
})
|
||||
|
||||
|
||||
test('Test Push URL', () => {
|
||||
//
|
||||
// These used to hit https://www.solidscribe.com over the live network, so the
|
||||
// suite needed internet access and the assertion depended on whatever that page
|
||||
// happened to serve. fetch is mocked against a fixture instead.
|
||||
//
|
||||
describe('Push URL', () => {
|
||||
|
||||
let url = 'https://www.solidscribe.com'
|
||||
beforeEach(() => {
|
||||
FetchMock.install()
|
||||
Robots.__resetCache()
|
||||
dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
|
||||
// Real code path, no politeness waiting and no robots lookups.
|
||||
ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false })
|
||||
})
|
||||
|
||||
return Attachment.pushUrl(newPushKey, url)
|
||||
.then(( results => {
|
||||
afterEach(() => {
|
||||
FetchMock.restore()
|
||||
dnsSpy.mockRestore()
|
||||
ScrapeQueue.__reset()
|
||||
})
|
||||
|
||||
return Attachment.textSearch(newUserId, 'scribe')
|
||||
test('scrapes a pushed url into a searchable attachment', () => {
|
||||
|
||||
}))
|
||||
.then((results) => {
|
||||
const url = 'https://www.solidscribe.com/'
|
||||
|
||||
expect(results.length == 1).toBe(true)
|
||||
FetchMock.route(url, () => FetchMock.html(`
|
||||
<html><head>
|
||||
<title>Solid Scribe</title>
|
||||
<meta property="og:description" content="An encrypted note taking application.">
|
||||
</head><body><p>x</p></body></html>
|
||||
`))
|
||||
|
||||
return Attachment.pushUrl(newPushKey, url)
|
||||
.then(() => ScrapeQueue.drain())
|
||||
.then(() => Attachment.textSearch(newUserId, 'scribe'))
|
||||
.then(results => {
|
||||
expect(results.length == 1).toBe(true)
|
||||
expect(results[0].text).toContain('Solid Scribe')
|
||||
})
|
||||
})
|
||||
|
||||
test('stores the description so it is searchable too', () => {
|
||||
|
||||
const url = 'https://www.solidscribe.com/encrypted'
|
||||
|
||||
FetchMock.route(url, () => FetchMock.html(`
|
||||
<html><head>
|
||||
<title>Encryption Page</title>
|
||||
<meta property="og:description" content="Notes are sealed with aes256gcm.">
|
||||
</head><body><p>x</p></body></html>
|
||||
`))
|
||||
|
||||
return Attachment.pushUrl(newPushKey, url)
|
||||
.then(() => ScrapeQueue.drain())
|
||||
.then(() => Attachment.textSearch(newUserId, 'aes256gcm'))
|
||||
.then(results => {
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
// The pushmebaby query param was previously passed through with no
|
||||
// validation whatsoever.
|
||||
test('refuses a pushed url pointing at loopback and makes no request', () => {
|
||||
|
||||
FetchMock.route(/.*/, () => FetchMock.html('<title>should never load</title>'))
|
||||
|
||||
return Attachment.pushUrl(newPushKey, 'http://127.0.0.1:3000/admin')
|
||||
.then(result => {
|
||||
expect(result).toBe(true)
|
||||
expect(FetchMock.callCount()).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
test('refuses a pushed url pointing at the cloud metadata endpoint', () => {
|
||||
|
||||
FetchMock.route(/.*/, () => FetchMock.html('<title>should never load</title>'))
|
||||
|
||||
return Attachment.pushUrl(newPushKey, 'http://169.254.169.254/latest/meta-data/')
|
||||
.then(() => {
|
||||
expect(FetchMock.callCount()).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
test('resolves truthy for an unknown pushkey without scraping', () => {
|
||||
|
||||
FetchMock.route(/.*/, () => FetchMock.html('<title>nope</title>'))
|
||||
|
||||
return Attachment.pushUrl('not-a-real-pushkey', 'https://example.com/')
|
||||
.then(result => {
|
||||
expect(result).toBe(true)
|
||||
expect(FetchMock.callCount()).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
// Previously this could hang the HTTP response indefinitely.
|
||||
test('returns within the budget when the server never responds', () => {
|
||||
|
||||
const url = 'https://www.solidscribe.com/slow'
|
||||
FetchMock.route(url, () => FetchMock.hang())
|
||||
|
||||
ScrapeQueue.configure({
|
||||
perHostMinSpacingMs: 0,
|
||||
perHostMaxSpacingMs: 0,
|
||||
checkRobots: false,
|
||||
pageTimeoutMs: 80,
|
||||
totalBudgetMs: 300,
|
||||
retries: 0,
|
||||
})
|
||||
|
||||
const startedAt = Date.now()
|
||||
|
||||
return Attachment.pushUrl(newPushKey, url)
|
||||
.then(result => {
|
||||
expect(result).toBe(true)
|
||||
expect(Date.now() - startedAt).toBeLessThan(5000)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
Reference in new issue
Block a user