* Added some better base information to site for scrapers

* Updated help text
* Refactored a lot of the scrape code into a SiteScrape helper
This commit is contained in:
Max G 2020-04-13 06:17:37 +00:00
parent a3fa4b0f3c
commit 3535f0cb24
7 changed files with 208 additions and 99 deletions

View File

@ -10,10 +10,23 @@
<meta name="theme-color" content="#000" />
<link rel="manifest" href="/api/static/assets/manifest.json">
<title>Notes</title>
<title>Solid Scribe - A Note Taking Website</title>
</head>
<body>
<div id="app"></div>
<div id="app">
<!-- placeholder data for scrapers with no JS -->
<h1>You have found a Solid Scribe</h1>
<img src="/api/static/assets/logo.svg" alt="logo">
<h1>Solid Scribe</h1>
<h2>A note application that respects your privacy.</h2>
<p>Take notes with a clean editor that works on desktop or mobile.</p>
<p>Search notes, links and files to find what you need.</p>
<p>Accessable everywhere.</p>
<p>Categorize notes with tags.</p>
<p>Share data with fellow users.</p>
<p>Encrypt notes for additional security.</p>
<b>This site requires Javascipt to run.</b>
</div>
<!-- built files will be auto injected -->
</body>
</html>

View File

@ -2,17 +2,16 @@
<div class="ui basic segment">
<div class="ui grid">
<div class="sixteen wide column">
<div class="ui text container">
<h2 class="ui dividing header">
Help
</h2>
</div>
<div class="sixteen wide column">
<div class="ui text container">
<!-- Content copied from note -->
<!-- https://www.solidscribe.com/#/notes/open/552 -->
<p><b>Quick Note</b><br></p><p>The Quick note feature was designed to allow rapid input to a single note. Rather than junking up all your notes with random links, numbers or haikus, you can put them all in one place. <br></p><p>All data pushed to the quick note can still be edited like a normal note.<br></p><p><br></p><p><b>Dark Theme</b><br></p><p>Dark theme was designed to minimize the amount of blue. Less blue entering your eyes is supposed to help you fall asleep.<br></p><p>Most things turn sepia and a filter is applied to images to make them more sepia.<br></p><p>Here is some good research on the topic: <a href="https://justgetflux.com/research.html">https://justgetflux.com/research.html</a><br></p><p><br></p><p><b>Password Protected Notes</b><br></p><p>Note protected with a password are encrypted. This means the data is scrambled and unreadable unless the correct password is used to decrypt them.<br></p><p>If a password is forgotten, it can never be recovered. Passwords are not saved for encrypted notes. If you lose the password to a protected note, that note text is lost. <br></p><p>Only the text of the note is protected. Tags, Files attached to the note, and the title of the note are still visible without a password. You can not search text in a password protected note. But you can search by the title.<br></p><p><br></p><p><b>Links in notes</b><br></p><p>Links put into notes are automatically scraped. This means the data from the link will be scanned to get an image and some text from the website to help make that link more accessible in the future. <br></p><p><br></p><p><b>Files in notes</b><br></p><p>Files can be uploaded to notes. If its an image, the picture will be put into the note.<br></p><p>Images added to notes will have the text pulled out so it can be searched (This isn't super accurate so don't rely to heavily on it.) The text can be updated at any time.<br></p><p><br></p><p><b>Deleting notes</b><br></p><p>When notes are deleted, none of the files related to the note are deleted.<br></p>
<p><b>Quick Note</b><br></p><p>The Quick note feature was designed to allow rapid input to a single note. Rather than junking up all your notes with random links, numbers or haikus, you can put them all in one place. <br></p><p>All data pushed to the quick note can still be edited like a normal note.<br></p><p><br></p><p><b>Dark Theme</b><br></p><p>Dark theme was designed to minimize the amount of blue. Less blue entering your eyes is supposed to help you fall asleep.<br></p><p>Most things turn sepia and a filter is applied to images to make them more sepia.<br></p><p>Here is some good research on the topic: <a href="https://justgetflux.com/research.html">https://justgetflux.com/research.html</a><br></p><p><br></p><p><b>Password Protected Notes</b><br></p><p>Note protected with a password are encrypted. This means the data is scrambled and unreadable unless the correct password is used to decrypt them.<br></p><p>If a password is forgotten, it can never be recovered. Passwords are not saved for encrypted notes. If you lose the password to a protected note, that note text is lost. <br></p><p>Only the text of the note is protected. Tags, Files attached to the note, and the title of the note are still visible without a password. You can not search text in a password protected note. But you can search by the title.<br></p><p><br></p><p><b>Links in notes</b><br></p><p>Links put into notes are automatically scraped. This means the data from the link will be scanned to get an image and some text from the website to help make that link more accessible in the future. <br></p><p><br></p><p><b>Files in notes</b><br></p><p>Files can be uploaded to notes. If its an image, the picture will be put into the note.<br></p><p>Images added to notes will have the text pulled out so it can be searched (This isn't super accurate so don't rely to heavily on it.) The text can be updated at any time.<br></p><p><br></p><p><b>Deleting notes</b><br></p><p>When<b> </b>notes are deleted, none of the files related to the note are deleted. <br></p><p><br></p><p><b>Daily Backups</b><br></p><p>All notes are backed up, every night, at midnight. If there is data loss, it can be restored from a backup. If you experience some sort of cataclysmic data loss please contact the system administrator for a copy of your data or restoration a restoration procedure. <br></p>
<!-- content copied from note -->
</div>

View File

@ -0,0 +1,151 @@
let SiteScrape = module.exports = {}
//
// $ = the cheerio scrape object
//
const removeWhitespace = /\s+/g
const commonWords = ['share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
SiteScrape.getTitle = ($) => {
let title = $('title').text().replace(removeWhitespace, " ")
return title
}
//Finds all urls in text, removes duplicates, makes sure they have https://
SiteScrape.getCleanUrls = (textBlock) => {
//Find all URLs in text
//@TODO - Use the process text library for this function
const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
let allUrls = textBlock.match(urlPattern)
if(allUrls == null){
return []
}
//Every URL needs HTTPS!!!
let foundUrls = []
allUrls.forEach( (item, index) => {
//add protocol if it is missing
if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
allUrls[index] = 'https://'+item
}
//convert http to https
if(item.indexOf('http://') >= 0){
allUrls[index] = item.replace('http://','https://')
}
})
//Remove all duplicates
foundUrls = [...new Set(allUrls)]
return foundUrls
}
//Site hostname with https:// eg: https://www.google.com
SiteScrape.getHostName = (url) => {
var hostname = 'https://'+(new URL(url)).hostname;
console.log('hostname', hostname)
return hostname
}
// URL for image that can be downloaded to represent website
SiteScrape.getDisplayImage = ($, url) => {
const hostname = SiteScrape.getHostName(url)
let metaImg = $('meta[property="og:image"]')
let shortcutIcon = $('link[rel="shortcut icon"]')
let favicon = $('link[rel="icon"]')
let randomImg = $('img')
console.log('----')
//Scrape metadata for page image
//Grab the first random image we find
if(randomImg && randomImg[0] && randomImg[0].attribs){
thumbnail = hostname + randomImg[0].attribs.src
console.log('random img '+thumbnail)
}
//Grab the favicon of the site
if(favicon && favicon[0] && favicon[0].attribs){
thumbnail = hostname + favicon[0].attribs.href
console.log('favicon '+thumbnail)
}
//Grab the shortcut icon
if(shortcutIcon && shortcutIcon[0] && shortcutIcon[0].attribs){
thumbnail = hostname + shortcutIcon[0].attribs.href
console.log('shortcut '+thumbnail)
}
//Grab the presentation image for the site
if(metaImg && metaImg[0] && metaImg[0].attribs){
thumbnail = metaImg[0].attribs.content
console.log('ogImg '+thumbnail)
}
console.log('-----')
return thumbnail
}
// Get all the site text and parse out the words that appear most
SiteScrape.getKeywords = ($) => {
let majorContent = ''
majorContent += $('[class*=content]').text()
.replace(removeWhitespace, " ") //Remove all whitespace
.replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,3000) //Limit to 3000 characters
.toLowerCase()
//Count frequency of each word in scraped text
let frequency = {}
majorContent.split(' ').forEach(word => {
if(commonWords.includes(word)){
return //Exclude certain words
}
if(!frequency[word]){
frequency[word] = 0
}
frequency[word]++
})
//Create a sortable array
var sortable = [];
for (var index in frequency) {
if(frequency[index] > 1){
sortable.push([index, frequency[index]]);
}
}
//Sort them by most used words in the list
sortable.sort(function(a, b) {
return b[1] - a[1];
});
let finalWords = []
for(let i=0; i<5; i++){
if(sortable[i] && sortable[i][0]){
finalWords.push(sortable[i][0])
}
}
if(finalWords.length > 0){
return 'Keywords: ' + finalWords.join(', ')
}
return ''
}
SiteScrape.getMainText = ($) => {}

View File

@ -128,6 +128,18 @@ app.use(function(req, res, next){
}
})
// Testing Area
// let att = require('@models/Attachment')
// let testUrl = 'https://dba.stackexchange.com/questions/23908/how-to-search-a-mysql-database-with-encrypted-fields'
// testUrl = 'https://www.solidscribe.com/#/'
// console.log('About to scrape: ', testUrl)
// att.processUrl(61, 3213, testUrl)
// .then(results => {
// console.log('Scrape happened')
// })
//
//
//Test
app.get(prefix, (req, res) => res.send('The api is running'))

View File

@ -1,5 +1,7 @@
let db = require('@config/database')
let SiteScrape = require('@helpers/SiteScrape')
let Attachment = module.exports = {}
const cheerio = require('cheerio')
@ -242,32 +244,8 @@ Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
Attachment.urlForNote(userId, noteId).then(attachments => {
//Find all URLs in text
//@TODO - Use the process text library for this function
const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
let allUrls = noteText.match(urlPattern)
if(allUrls == null){
allUrls = []
}
//Every URL needs HTTPS!!!
let foundUrls = []
allUrls.forEach( (item, index) => {
//Every URL should have HTTPS
if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
allUrls[index] = 'https://'+item
}
//URLs should all have HTTPS!!!
if(item.indexOf('http://') >= 0){
allUrls[index] = item.replace('http://','https://')
}
})
//Remove all duplicates
foundUrls = [...new Set(allUrls)]
//Pull all the URLs out of the text
let foundUrls = SiteScrape.getCleanUrls(noteText)
//Go through each saved URL, remove new URLs from saved URLs
//If a URL is not found, delete it
@ -386,14 +364,6 @@ Attachment.processUrl = (userId, noteId, url) => {
return new Promise((resolve, reject) => {
const excludeWords = ['share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
var removeWhitespace = /\s+/g
const options = {
uri: url,
@ -428,70 +398,33 @@ Attachment.processUrl = (userId, noteId, url) => {
})
.then($ => {
//Clear timeout that would end this function
clearTimeout(requestTimeout)
var desiredSearchText = ''
let pageTitle = $('title').text().replace(removeWhitespace, " ")
desiredSearchText += pageTitle + "\n"
// let header = $('h1').text().replace(removeWhitespace, " ")
// desiredSearchText += header + "\n"
//Scrape metadata for page image
let metadata = $('meta[property="og:image"]')
if(metadata && metadata[0] && metadata[0].attribs){
thumbnail = metadata[0].attribs.content
}
const pageTitle = SiteScrape.getTitle($)
const hostname = SiteScrape.getHostName(url)
let majorContent = ''
majorContent += $('[class*=content]').text()
.replace(removeWhitespace, " ") //Remove all whitespace
.replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,3000)
.toLowerCase()
majorContent += $('[id*=content]').text().replace(removeWhitespace, " ")
.replace(removeWhitespace, " ") //Remove all whitespace
.replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,3000) //Limit characters
.toLowerCase()
const thumbnail = SiteScrape.getDisplayImage($, url)
//Count frequency of each word in scraped text
let frequency = {}
majorContent.split(' ').forEach(word => {
if(excludeWords.includes(word)){
return //Exclude certain words
}
if(!frequency[word]){
frequency[word] = 0
}
frequency[word]++
const keywords = SiteScrape.getKeywords($)
var desiredSearchText = ''
desiredSearchText += pageTitle + "\n"
desiredSearchText += keywords
console.log({
pageTitle,
hostname,
thumbnail,
keywords
})
//Create a sortable array
var sortable = [];
for (var index in frequency) {
if(frequency[index] > 1){
sortable.push([index, frequency[index]]);
}
}
//Sort them by most used words in the list
sortable.sort(function(a, b) {
return b[1] - a[1];
});
let finalWords = []
for(let i=0; i<5; i++){
if(sortable[i] && sortable[i][0]){
finalWords.push(sortable[i][0])
}
}
if(finalWords.length > 0){
desiredSearchText += 'Keywords: ' + finalWords.join(', ')
}
// throw new Error('Ending this function early.')
// console.log('TexT Scraped')
@ -532,9 +465,10 @@ Attachment.processUrl = (userId, noteId, url) => {
})
.catch(error => {
console.log('Issue with scrape')
// console.log('Scrape pooped out')
// console.log('Issue with scrape')
console.log(error)
resolve('')
// resolve('')
})
requestTimeout = setTimeout( () => {

View File

@ -167,10 +167,10 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived,
return new Promise((resolve, reject) => {
//Prevent note loss if it saves with empty text
if(ProcessText.removeHtml(noteText) == ''){
//if(ProcessText.removeHtml(noteText) == ''){
// console.log('Not saving empty note')
// resolve(false)
}
//}
const now = Math.round((+new Date)/1000)

View File

@ -4,7 +4,7 @@ var multer = require('multer')
var upload = multer({ dest: '../staticFiles/' }) //@TODO make this a global value
let router = express.Router()
let Attachment = require('@models/Attachment');
let Attachment = require('@models/Attachment')
let Note = require('@models/Note')
let userId = null