Max/SolidScribe
1
0
Fork 0
Code Issues 24 Pull Requests Releases Wiki Activity

Gigantic Update

Browse Source 
* Migrated manual tests to jest and started working on better coverage
* Added a bookmarklet and push key generation tool allowing URL pushing from bookmarklets
* Updated web scraping with tons of bug fixes
* Updated attachments page to handle new push links
* Aggressive note change checking, if patches get out of sync, server overwrites bad updates.
This commit is contained in:
Max
2023-10-17 19:46:14 +00:00
parent b5ef64485f
commit 276a72b4ce
25 changed files with 7442 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
server/models/Attachment.js
View File

@@ -1,6 +1,7 @@
let db = require('@config/database')
let SiteScrape = require('@helpers/SiteScrape')
const cs = require('@helpers/CryptoString')
let Attachment = module.exports = {}
@@ -47,13 +48,15 @@ Attachment.textSearch = (userId, searchTerm) => {
}
Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeShared) => {
console.log([userId, noteId, attachmentType, offset, setSize, includeShared])
return new Promise((resolve, reject) => {
let params = [userId]
let query = `
SELECT attachment.*, note.share_user_id FROM attachment
JOIN note ON (attachment.note_id = note.id)
WHERE attachment.user_id = ? AND visible = 1 `
LEFT JOIN note ON (attachment.note_id = note.id)
WHERE attachment.user_id = ? AND visible = 1
`
if(noteId && noteId > 0){
//
@@ -76,6 +79,11 @@ Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeSha
query += `AND note.archived = ${ attachmentType == 'archived' ? '1':'0' } `
query += `AND note.trashed = ${ attachmentType == 'trashed' ? '1':'0' } `
if(!attachmentType){
// Null note ID means it was pushed by bookmarklet
query += 'OR attachment.note_id IS NULL '
}
}
@@ -102,18 +110,6 @@ Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeSha
})
}
//Returns all attachments
Attachment.forNote = (userId, noteId) => {
return new Promise((resolve, reject) => {
db.promise()
.query(`SELECT * FROM attachment WHERE user_id = ? AND note_id = ? AND visible = 1 ORDER BY last_indexed DESC;`, [userId, noteId])
.then((rows, fields) => {
resolve(rows[0]) //Return all attachments found by query
})
.catch(console.log)
})
}
Attachment.urlForNote = (userId, noteId) => {
return new Promise((resolve, reject) => {
db.promise()
@@ -189,6 +185,7 @@ Attachment.delete = (userId, attachmentId, urlDelete = false) => {
.catch(console.log)
}
})
.catch(console.log)
})
}
@@ -305,9 +302,13 @@ Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
//Once everything is done being scraped, emit new attachment events
SocketIo.to(userId).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(userId).emit('update_note_attachments')
solrAttachmentText += freshlyScrapedText
resolve(solrAttachmentText)
})
.catch(console.log)
})
})
}
@@ -335,9 +336,13 @@ Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
//All URLs have been scraped, return data
if(processedCount == foundUrls.length){
resolve(scrapedText)
console.log('All urls scraped')
return resolve(scrapedText)
}
})
.catch(error => {
console.log('Site Scrape error', error)
})
})
})
}
@@ -347,8 +352,8 @@ Attachment.downloadFileFromUrl = (url) => {
return new Promise((resolve, reject) => {
if(url == null || url == undefined || url == ''){
resolve(null)
if(!url){
return resolve(null)
}
const random = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
@@ -356,8 +361,7 @@ Attachment.downloadFileFromUrl = (url) => {
let fileName = random+'_scrape'
let thumbPath = 'thumb_'+fileName
console.log('Scraping image url')
console.log(url)
console.log('Scraping image url', url)
console.log('Getting ready to scrape ', url)
@@ -395,7 +399,7 @@ Attachment.downloadFileFromUrl = (url) => {
Attachment.processUrl = (userId, noteId, url) => {
const scrapeTime = 20*1000;
const scrapeTime = 5*1000;
return new Promise((resolve, reject) => {
@@ -450,9 +454,10 @@ Attachment.processUrl = (userId, noteId, url) => {
var desiredSearchText = ''
desiredSearchText += pageTitle
if(keywords){
desiredSearchText += "\n" + keywords
desiredSearchText += "\n " + keywords
}
console.log('Results from site scrape-------------')
console.log({
pageTitle,
hostname,
@@ -502,48 +507,142 @@ Attachment.processUrl = (userId, noteId, url) => {
})
.catch(error => {
// console.log('Scrape pooped out')
// console.log('Issue with scrape')
console.log(error)
// resolve('')
console.log('Scrape pooped out')
console.log('Issue with scrape', error.statusCode)
clearTimeout(requestTimeout)
return resolve('No site text')
})
requestTimeout = setTimeout( () => {
console.log('Cancel the request, its taking to long.')
request.cancel()
desiredSearchText = 'No Description for -> '+url
created = Math.round((+new Date)/1000)
db.promise()
.query(`UPDATE attachment SET
text = ?,
last_indexed = ?,
WHERE id = ?
`, [desiredSearchText, created, insertedId])
.then((rows, fields) => {
resolve(desiredSearchText) //Return found text
})
.catch(console.log)
//Create attachment in DB with scrape text and provided data
// db.promise()
// .query(`INSERT INTO attachment
// (note_id, user_id, attachment_type, text, url, last_indexed)
// VALUES (?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created])
// .then((rows, fields) => {
// resolve(desiredSearchText) //Return found text
// })
// .catch(console.log)
return resolve('Request Timeout')
}, scrapeTime )
})
}
Attachment.generatePushKey = (userId) => {}
Attachment.generatePushKey = (userId) => {
return new Promise((resolve, reject) => {
Attachment.deletePushKey = (userId) => {}
db.promise()
.query("SELECT pushkey FROM user WHERE id = ? LIMIT 1", [userId])
.then((rows, fields) => {
Attachment.getPushkey = (userId) => {}
const pushKey = rows[0][0].pushkey
// push key exists
if(pushKey && pushKey.length > 0){
Attachment.pushUrl = (userId) => {}
return resolve(pushKey)
} else {
// generate and save a new key
const newPushKey = cs.createSmallSalt()
db.promise()
.query('UPDATE user SET pushkey = ? WHERE id = ? LIMIT 1', [newPushKey,userId])
.then((rows, fields) => {
return resolve(newPushKey)
})
}
})
})
}
Attachment.deletePushKey = (userId) => {
return new Promise((resolve, reject) => {
db.promise()
.query('UPDATE user SET pushkey = null WHERE id = ? LIMIT 1', [userId])
.then((rows, fields) => {
return resolve(rows[0].affectedRows == 1)
})
})
}
Attachment.getPushkeyBookmarklet = (userId) => {
return new Promise((resolve, reject) => {
Attachment.generatePushKey(userId)
.then( pushKey => {
let bookmarklet = Attachment.generateBookmarkletText(pushKey)
return resolve(bookmarklet)
})
})
}
Attachment.pushUrl = (pushkey,url) => {
return new Promise((resolve, reject) => {
let userId = null
pushkey = pushkey.replace(/ /g, '+')
db.promise()
.query("SELECT id FROM user WHERE pushkey = ? LIMIT 1", [pushkey])
.then((rows, fields) => {
if(rows[0].length == 0){
return resolve(true)
}
userId = rows[0][0].id
return Attachment.scrapeUrlsCreateAttachments(userId, null, [url])
})
.then(() => {
if(typeof SocketIo != 'undefined'){
//Once everything is done being scraped, emit new attachment events
SocketIo.to(userId).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(userId).emit('update_note_attachments')
}
return resolve(true)
})
.catch(console.log)
})
}
Attachment.generateBookmarkletText = (pushKey) => {
const endpoint = '/api/public/pushmebaby'
let url = 'https://www.solidscribe.com' + endpoint
if(process.env.NODE_ENV === 'development'){
// url = 'https://192.168.1.164' + endpoint
}
// Terminate each line with a semi-colon, super important, since spaces are removed.
// document.getElementById(id).remove();
url += '?pushkey='+encodeURIComponent(pushKey)
const bookmarkletV3 = `
javascript: (() => {
var p = encodeURIComponent(window.location.href);
var n = "`+url+`&url="+p;
window.open(n, '_blank', 'noopener=noopener');
window.focus();
var k = document.createElement("div");
k.setAttribute("style", "position:fixed;right:10px;top:10px;z-index:222222;border-radius:4px;font-size:1.3em;padding:20px 15px;background: #8f51be;color:white;");
k.innerHTML = "Posted URL to your Solid Scribe account";
document.body.appendChild(k);
setTimeout(()=>{
k.remove();
},5000);
})();
`
return bookmarkletV3
.replace(/\t|\r|\n/gm, "") // Remove tabs, new lines, returns
.replace(/\s+/g, ' ') // remove double spaces
.trim()
}

35
server/models/Note.js
View File

@@ -17,6 +17,7 @@ const fs = require('fs')
const gm = require('gm')
Note.test = (userId, masterKey, printResults) => {
return false;
return new Promise((resolve, reject) => {
@@ -162,6 +163,10 @@ Note.test = (userId, masterKey, printResults) => {
return resolve('Test: Complete ---')
})
.catch(error => {
console.log(error)
return reject(error)
})
})
}
@@ -193,7 +198,7 @@ Note.create = (userId, noteTitle = '', noteText = '', masterKey) => {
})
.then((rows, fields) => {
if(SocketIo){
if(typeof SocketIo != 'undefined'){
SocketIo.to(userId).emit('new_note_created', rows[0].insertId)
}
@@ -341,7 +346,7 @@ Note.reindex = (userId, masterKey, removeId = null) => {
setTimeout(() => {
if(masterKey == null || note.salt == null){
console.log('Error indexing note', note.id)
console.log('Error indexing note - master key or salt missing', note.id)
return resolve(true)
}
@@ -390,13 +395,13 @@ Note.reindex = (userId, masterKey, removeId = null) => {
return Promise.all(reindexQueue)
})
.then(rawSearchIndex => {
.then(updatePromiseResults => {
const created = Math.round((+new Date)/1000)
const jsonSearchIndex = JSON.stringify(searchIndex)
const encryptedJsonIndex = cs.encrypt(masterKey, searchIndexSalt, jsonSearchIndex)
return db.promise().query("UPDATE user_encrypted_search_index SET `index` = ?, `last_update` = ? WHERE (`user_id` = ?) LIMIT 1",
db.promise().query("UPDATE user_encrypted_search_index SET `index` = ?, `last_update` = ? WHERE (`user_id` = ?) LIMIT 1",
[encryptedJsonIndex, created, userId])
.then((rows, fields) => {
@@ -406,6 +411,7 @@ Note.reindex = (userId, masterKey, removeId = null) => {
.then((rows, fields) => {
// console.log('Indexd Note Count: ' + rows[0]['affectedRows'])
// @TODO - Return number of reindexed notes
resolve(true)
})
@@ -507,13 +513,13 @@ Note.update = (userId, noteId, noteText, noteTitle, color, pinned, archived, has
})
.then((rows, fields) => {
if(SocketIo){
if(typeof SocketIo != 'undefined'){
SocketIo.to(userId).emit('new_note_text_saved', {noteId, hash})
//Async attachment reindex
Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
}
//Async attachment reindex
Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
//Send back updated response
resolve(rows[0])
})
@@ -739,12 +745,13 @@ Note.get = (userId, noteId, masterKey) => {
const nowTime = Math.round((+new Date)/1000)
db.promise().query(`UPDATE note SET opened = ? WHERE (id = ?)`, [nowTime, noteId])
//Return note data
// delete noteData.salt //remove salt from return data
// delete noteData.encrypted_share_password_key
noteData.lockedOut = noteLockedOut
resolve(noteData)
.then(results => {
//Return note data
// delete noteData.salt //remove salt from return data
// delete noteData.encrypted_share_password_key
noteData.lockedOut = noteLockedOut
resolve(noteData)
})
})
.catch(error => {

83
server/models/User.js
View File

@@ -9,7 +9,8 @@ const speakeasy = require('speakeasy')
let User = module.exports = {}
const version = '3.6.3'
const version = '3.8.0'
// 3.7.3 - diff/patch update
//Login a user, if that user does not exist create them
//Issues login token
@@ -552,6 +553,12 @@ User.revokeActiveSessions = (userId, sessionId) => {
User.deleteUser = (userId, password) => {
if(!userId || !password){
return new Promise((resolve, reject) => {
return resolve('Missing User ID or Password. No Action Taken.')
})
}
//Verify user is correct by decryptig master key with password
let deletePromises = []
@@ -583,78 +590,4 @@ User.deleteUser = (userId, password) => {
//Remove all note attachments and files
return Promise.all(deletePromises)
}
User.keyPairTest = (testUserName = 'genMan', password = '1', printResults) => {
return new Promise((resolve, reject) => {
let masterKey = null
let testUserId = null
const randomUsername = Math.random().toString(36).substring(2, 15);
const randomPassword = '1'
const secondPassword = '2'
User.register(testUserName, password)
.then( ({ token, userId }) => {
testUserId = userId
if(printResults) console.log('Test: Register User '+testUserName+' - Pass')
return User.getMasterKey(testUserId, password)
})
.then(newMasterKey => {
masterKey = newMasterKey
if(printResults) console.log('Test: Generate/Decrypt Master Key - Pass')
return User.generateKeypair(testUserId, masterKey)
})
.then(({publicKey, privateKey}) => {
const publicKeyMessage = 'Test: Public key decrypt - Pass'
const privateKeyMessage = 'Test: Private key decrypt - Pass'
//Encrypt Message with private Key
const privateKeyEncrypted = crypto.privateEncrypt(privateKey, Buffer.from(privateKeyMessage, 'utf8')).toString('base64')
const decryptedPrivate = crypto.publicDecrypt(publicKey, Buffer.from(privateKeyEncrypted, 'base64'))
//Conver back to a string
if(printResults) console.log(decryptedPrivate.toString('utf8'))
//Encrypt with public key
const pubEncrMsc = crypto.publicEncrypt(publicKey, Buffer.from(publicKeyMessage, 'utf8')).toString('base64')
const publicDeccryptMessage = crypto.privateDecrypt(privateKey, Buffer.from(pubEncrMsc, 'base64') )
//Convert it back to string
if(printResults) console.log(publicDeccryptMessage.toString('utf8'))
return User.login(testUserName, password)
})
.then( ({token, userId}) => {
if(printResults) console.log('Test: Login New User - Pass')
return User.changePassword(testUserId, randomPassword, secondPassword)
})
.then(passwordChangeResults => {
if(printResults) console.log('Test: Password Change - ', passwordChangeResults?'Pass':'Fail')
return User.login(testUserName, secondPassword)
})
.then(reLogin => {
if(printResults) console.log('Test: Login With new Password - Pass')
return User.getMasterKey(testUserId, secondPassword)
})
.then(newMasterKey => {
masterKey = newMasterKey
resolve({testUserId, masterKey})
})
})
}