Max/SolidScribe
1
0 Fork 0
Code Issues 24 Pull Requests Releases Wiki Activity

Gigantic Update

Browse Source 
* Migrated manual tests to jest and started working on better coverage
* Added a bookmarklet and push key generation tool allowing URL pushing from bookmarklets
* Updated web scraping with tons of bug fixes
* Updated attachments page to handle new push links
* Aggressive note change checking, if patches get out of sync, server overwrites bad updates.
This commit is contained in:
Max committed 2023-10-17 19:46:14 +00:00
1 parent b5ef64485f
commit 276a72b4ce
25 files changed
+7442 -245
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/models/Attachment.js
+153 -54
View File
@@ -1,6 +1,7 @@
let db = require('@config/database')
let SiteScrape = require('@helpers/SiteScrape')
const cs = require('@helpers/CryptoString')
let Attachment = module.exports = {}
@@ -47,13 +48,15 @@ Attachment.textSearch = (userId, searchTerm) => {
}
Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeShared) => {
console.log([userId, noteId, attachmentType, offset, setSize, includeShared])
return new Promise((resolve, reject) => {
let params = [userId]
let query = `
SELECT attachment.*, note.share_user_id FROM attachment
JOIN note ON (attachment.note_id = note.id)
WHERE attachment.user_id = ? AND visible = 1 `
LEFT JOIN note ON (attachment.note_id = note.id)
WHERE attachment.user_id = ? AND visible = 1
`
if(noteId && noteId > 0){
//
@@ -76,6 +79,11 @@ Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeSha
query += `AND note.archived = ${ attachmentType == 'archived' ? '1':'0' } `
query += `AND note.trashed = ${ attachmentType == 'trashed' ? '1':'0' } `
if(!attachmentType){
// Null note ID means it was pushed by bookmarklet
query += 'OR attachment.note_id IS NULL '
}
}
@@ -102,18 +110,6 @@ Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeSha
})
}
//Returns all attachments
Attachment.forNote = (userId, noteId) => {
return new Promise((resolve, reject) => {
db.promise()
.query(`SELECT * FROM attachment WHERE user_id = ? AND note_id = ? AND visible = 1 ORDER BY last_indexed DESC;`, [userId, noteId])
.then((rows, fields) => {
resolve(rows[0]) //Return all attachments found by query
})
.catch(console.log)
})
}
Attachment.urlForNote = (userId, noteId) => {
return new Promise((resolve, reject) => {
db.promise()
@@ -189,6 +185,7 @@ Attachment.delete = (userId, attachmentId, urlDelete = false) => {
.catch(console.log)
}
})
.catch(console.log)
})
}
@@ -305,9 +302,13 @@ Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
//Once everything is done being scraped, emit new attachment events
SocketIo.to(userId).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(userId).emit('update_note_attachments')
solrAttachmentText += freshlyScrapedText
resolve(solrAttachmentText)
})
.catch(console.log)
})
})
}
@@ -335,9 +336,13 @@ Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
//All URLs have been scraped, return data
if(processedCount == foundUrls.length){
resolve(scrapedText)
console.log('All urls scraped')
return resolve(scrapedText)
}
})
.catch(error => {
console.log('Site Scrape error', error)
})
})
})
}
@@ -347,8 +352,8 @@ Attachment.downloadFileFromUrl = (url) => {
return new Promise((resolve, reject) => {
if(url == null || url == undefined || url == ''){
resolve(null)
if(!url){
return resolve(null)
}
const random = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
@@ -356,8 +361,7 @@ Attachment.downloadFileFromUrl = (url) => {
let fileName = random+'_scrape'
let thumbPath = 'thumb_'+fileName
console.log('Scraping image url')
console.log(url)
console.log('Scraping image url', url)
console.log('Getting ready to scrape ', url)
@@ -395,7 +399,7 @@ Attachment.downloadFileFromUrl = (url) => {
Attachment.processUrl = (userId, noteId, url) => {
const scrapeTime = 20*1000;
const scrapeTime = 5*1000;
return new Promise((resolve, reject) => {
@@ -450,9 +454,10 @@ Attachment.processUrl = (userId, noteId, url) => {
var desiredSearchText = ''
desiredSearchText += pageTitle
if(keywords){
desiredSearchText += "\n" + keywords
desiredSearchText += "\n " + keywords
}
console.log('Results from site scrape-------------')
console.log({
pageTitle,
hostname,
@@ -502,48 +507,142 @@ Attachment.processUrl = (userId, noteId, url) => {
})
.catch(error => {
// console.log('Scrape pooped out')
// console.log('Issue with scrape')
console.log(error)
// resolve('')
console.log('Scrape pooped out')
console.log('Issue with scrape', error.statusCode)
clearTimeout(requestTimeout)
return resolve('No site text')
})
requestTimeout = setTimeout( () => {
console.log('Cancel the request, its taking to long.')
request.cancel()
desiredSearchText = 'No Description for -> '+url
created = Math.round((+new Date)/1000)
db.promise()
.query(`UPDATE attachment SET
text = ?,
last_indexed = ?,
WHERE id = ?
`, [desiredSearchText, created, insertedId])
.then((rows, fields) => {
resolve(desiredSearchText) //Return found text
})
.catch(console.log)
//Create attachment in DB with scrape text and provided data
// db.promise()
// .query(`INSERT INTO attachment
// (note_id, user_id, attachment_type, text, url, last_indexed)
// VALUES (?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created])
// .then((rows, fields) => {
// resolve(desiredSearchText) //Return found text
// })
// .catch(console.log)
return resolve('Request Timeout')
}, scrapeTime )
})
}
Attachment.generatePushKey = (userId) => {}
Attachment.generatePushKey = (userId) => {
return new Promise((resolve, reject) => {
Attachment.deletePushKey = (userId) => {}
db.promise()
.query("SELECT pushkey FROM user WHERE id = ? LIMIT 1", [userId])
.then((rows, fields) => {
Attachment.getPushkey = (userId) => {}
const pushKey = rows[0][0].pushkey
// push key exists
if(pushKey && pushKey.length > 0){
Attachment.pushUrl = (userId) => {}
return resolve(pushKey)
} else {
// generate and save a new key
const newPushKey = cs.createSmallSalt()
db.promise()
.query('UPDATE user SET pushkey = ? WHERE id = ? LIMIT 1', [newPushKey,userId])
.then((rows, fields) => {
return resolve(newPushKey)
})
}
})
})
}
Attachment.deletePushKey = (userId) => {
return new Promise((resolve, reject) => {
db.promise()
.query('UPDATE user SET pushkey = null WHERE id = ? LIMIT 1', [userId])
.then((rows, fields) => {
return resolve(rows[0].affectedRows == 1)
})
})
}
Attachment.getPushkeyBookmarklet = (userId) => {
return new Promise((resolve, reject) => {
Attachment.generatePushKey(userId)
.then( pushKey => {
let bookmarklet = Attachment.generateBookmarkletText(pushKey)
return resolve(bookmarklet)
})
})
}
Attachment.pushUrl = (pushkey,url) => {
return new Promise((resolve, reject) => {
let userId = null
pushkey = pushkey.replace(/ /g, '+')
db.promise()
.query("SELECT id FROM user WHERE pushkey = ? LIMIT 1", [pushkey])
.then((rows, fields) => {
if(rows[0].length == 0){
return resolve(true)
}
userId = rows[0][0].id
return Attachment.scrapeUrlsCreateAttachments(userId, null, [url])
})
.then(() => {
if(typeof SocketIo != 'undefined'){
//Once everything is done being scraped, emit new attachment events
SocketIo.to(userId).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(userId).emit('update_note_attachments')
}
return resolve(true)
})
.catch(console.log)
})
}
Attachment.generateBookmarkletText = (pushKey) => {
const endpoint = '/api/public/pushmebaby'
let url = 'https://www.solidscribe.com' + endpoint
if(process.env.NODE_ENV === 'development'){
// url = 'https://192.168.1.164' + endpoint
}
// Terminate each line with a semi-colon, super important, since spaces are removed.
// document.getElementById(id).remove();
url += '?pushkey='+encodeURIComponent(pushKey)
const bookmarkletV3 = `
javascript: (() => {
var p = encodeURIComponent(window.location.href);
var n = "`+url+`&url="+p;
window.open(n, '_blank', 'noopener=noopener');
window.focus();
var k = document.createElement("div");
k.setAttribute("style", "position:fixed;right:10px;top:10px;z-index:222222;border-radius:4px;font-size:1.3em;padding:20px 15px;background: #8f51be;color:white;");
k.innerHTML = "Posted URL to your Solid Scribe account";
document.body.appendChild(k);
setTimeout(()=>{
k.remove();
},5000);
})();
`
return bookmarkletV3
.replace(/\t|\r|\n/gm, "") // Remove tabs, new lines, returns
.replace(/\s+/g, ' ') // remove double spaces
.trim()
}