SolidScribe/server/models/Note.js
Max G 1b14a8fd31 Added rate limiting and server security
Ton of little visual style tweaks and little up improvements for mobile
2020-03-26 04:45:23 +00:00

790 lines
22 KiB
JavaScript

let db = require('@config/database')
let Tags = require('@models/Tag')
let Attachment = require('@models/Attachment')
let ShareNote = require('@models/ShareNote')
let ProcessText = require('@helpers/ProcessText')
const DiffMatchPatch = require('@helpers/DiffMatchPatch')
const cs = require('@helpers/CryptoString')
const rp = require('request-promise');
const fs = require('fs')
let Note = module.exports = {}
const gm = require('gm')
// --------------
Note.migrateNoteTextToNewTable = () => {
return new Promise((resolve, reject) => {
db.promise()
.query('SELECT id, text FROM note WHERE note_raw_text_id IS NULL')
.then((rows, fields) => {
rows[0].forEach( ({id, text}) => {
db.promise()
.query('INSERT INTO note_raw_text (text) VALUES (?)', [text])
.then((rows, fields) => {
db.promise()
.query(`UPDATE note SET note_raw_text_id = ? WHERE (id = ?)`, [rows[0].insertId, id])
.then((rows, fields) => {
return 'Nice'
})
})
})
resolve('Its probably running... :-D')
})
})
}
Note.fixAttachmentThumbnails = () => {
const filePath = '../staticFiles/'
db.promise()
.query(`SELECT * FROM attachment WHERE file_location NOT LIKE "%.%"`)
.then( (rows, fields) => {
rows[0].forEach(line => {
const rawFilename = line['file_location']
const goodFileName = rawFilename+'.jpg'
//Rename file to have jpg extension, create thumbnail, update database
fs.rename(filePath+rawFilename, filePath+goodFileName, (err) => {
db.promise()
.query(`UPDATE attachment SET file_location = ? WHERE id = ?`,[goodFileName, line['id'] ])
.then( (rows, fields) => {
gm(filePath+goodFileName)
.resize(550) //Resize to width of 550 px
.quality(75) //compression level 0 - 100 (best)
.write(filePath + 'thumb_'+goodFileName, function (err) {
console.log('Done for -> ', goodFileName)
})
})
})
})
})
}
Note.stressTest = () => {
return new Promise((resolve, reject) => {
db.promise()
.query(`
SELECT text FROM note;
`)
.then((rows, fields) => {
console.log()
rows[0].forEach(item => {
Note.create(68, item['text'])
})
resolve(true)
})
.catch(console.log)
})
}
// --------------
Note.create = (userId, noteTitle, noteText, quickNote = 0, ) => {
return new Promise((resolve, reject) => {
if(userId == null || userId < 10){ reject('User Id required to create note') }
const created = Math.round((+new Date)/1000)
db.promise()
.query(`INSERT INTO note_raw_text (text, title, updated) VALUE (?, ?, ?)`, [noteText, noteTitle, created])
.then( (rows, fields) => {
const rawTextId = rows[0].insertId
return db.promise()
.query('INSERT INTO note (user_id, note_raw_text_id, created, quick_note) VALUES (?,?,?,?)',
[userId, rawTextId, created, quickNote])
})
.then((rows, fields) => {
// Indexing is done on save
resolve(rows[0].insertId) //Only return the new note ID when creating a new note
})
.catch(console.log)
})
}
Note.reindex = (userId, noteId) => {
return new Promise((resolve, reject) => {
Note.get(userId, noteId)
.then(note => {
let noteText = note.text
if(note.encrypted == 1){
noteText = '' //Don't put note text in encrypted notes
}
//
// Update Solr index
//
Tags.string(userId, noteId)
.then(tagString => {
const fullText = note.title + ' ' + ProcessText.removeHtml(noteText) +' '+ tagString
db.promise()
.query(`
INSERT INTO note_text_index (note_id, user_id, text)
VALUES (?,?,?)
ON DUPLICATE KEY UPDATE text = ?
`, [noteId, userId, fullText, fullText])
.then((rows, fields) => {
resolve(true)
})
.catch(console.log)
})
})
})
}
Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, password = '', passwordHint = '') => {
return new Promise((resolve, reject) => {
//Prevent note loss if it saves with empty text
if(ProcessText.removeHtml(noteText) == ''){
// console.log('Not saving empty note')
// resolve(false)
}
const now = Math.round((+new Date)/1000)
db.promise()
.query(`
SELECT note_raw_text_id, salt FROM note
JOIN note_raw_text ON note_raw_text_id = note_raw_text.id
WHERE note.id = ? AND user_id = ?`, [noteId, userId])
.then((rows, fields) => {
const textId = rows[0][0]['note_raw_text_id']
let salt = rows[0][0]['salt']
//If password is removed, remove salt. generate a new one next time its encrypted
if(password.length == 0){
salt = null
}
//If a password is set, create a salt
if(password.length > 3 && !salt){
salt = cs.createSalt()
//Save password hint on first encryption
if(passwordHint.length > 0){
db.promise().query('UPDATE note_raw_text SET password_hint = ? WHERE id = ?', [passwordHint, textId])
}
}
//Encrypt note text if proper data is setup
if(password.length > 3 && salt.length > 1000){
noteText = cs.encrypt(password, salt, noteText)
//
// @TODO - Do note save data if encryption goes wrong, do some validation
//
}
//Update Note text
return db.promise()
.query('UPDATE note_raw_text SET text = ?, title = ?, updated = ?, salt = ? WHERE id = ?', [noteText, noteTitle, now, salt, textId])
})
.then( (rows, fields) => {
const encrypted = password.length > 3 ? 1:0
//Update other note attributes
return db.promise()
.query('UPDATE note SET pinned = ?, archived = ?, color = ?, encrypted = ? WHERE id = ? AND user_id = ? LIMIT 1',
[pinned, archived, color, encrypted, noteId, userId])
})
.then((rows, fields) => {
//Async solr note reindex
Note.reindex(userId, noteId)
//Async attachment reindex
Attachment.scanTextForWebsites(io, userId, noteId, noteText)
//Send back updated response
resolve(rows[0])
})
.catch(console.log)
})
}
Note.setPinned = (userId, noteId, pinnedBoolean) => {
return new Promise((resolve, reject) => {
const pinned = pinnedBoolean ? 1:0
//Update other note attributes
return db.promise()
.query('UPDATE note SET pinned = ? WHERE id = ? AND user_id = ? LIMIT 1',
[pinned, noteId, userId])
.then((rows, fields) => {
resolve(true)
})
})
}
Note.setArchived = (userId, noteId, archivedBoolead) => {
return new Promise((resolve, reject) => {
const archived = archivedBoolead ? 1:0
//Update other note attributes
return db.promise()
.query('UPDATE note SET archived = ? WHERE id = ? AND user_id = ? LIMIT 1',
[archived, noteId, userId])
.then((rows, fields) => {
resolve(true)
})
})
}
//
// Delete a note and all its remaining parts
//
Note.delete = (userId, noteId) => {
return new Promise((resolve, reject) => {
//
// Delete, note, text, search index and associated tags
// Leave the attachments, they can be deleted on their own
// Leave Tags, their text is shared
let rawTextId = null
let noteTextCount = 0
// Lookup the note text ID, we need this to count usages
db.promise()
.query('SELECT note_raw_text_id FROM note WHERE id = ? AND user_id = ?', [noteId, userId])
.then((rows, fields) => {
//Save the raw text ID
rawTextId = rows[0][0]['note_raw_text_id']
return db.promise()
.query('SELECT count(id) as count FROM note WHERE note_raw_text_id = ?', [rawTextId])
})
.then((rows, fields) => {
//Save the number of times the note is used
noteTextCount = rows[0][0]['count']
//Don't delete text if its shared
if(noteTextCount == 1){
//If text is only used on one note, delete it (its not shared)
return db.promise()
.query('SELECT count(id) as count FROM note WHERE note_raw_text_id = ?', [rawTextId])
} else {
return new Promise((resolve, reject) => {
resolve(true)
})
}
})
.then( results => {
// Delete Note entry for this user.
return db.promise()
.query('DELETE FROM note WHERE note.id = ? AND note.user_id = ?', [noteId, userId])
})
.then((rows, fields) => {
// Delete search index
return db.promise()
.query('DELETE FROM note_text_index WHERE note_text_index.note_id = ? AND note_text_index.user_id = ?', [noteId,userId])
})
.then((rows, fields) => {
// delete tags
return db.promise()
.query('DELETE FROM note_tag WHERE note_tag.note_id = ? AND note_tag.user_id = ?', [noteId,userId])
})
.then((rows, fields) => {
//IF there are nots with a matching raw text id, we want to under their share status
db.promise().query('SELECT id FROM note WHERE note_raw_text_id = ?',[rawTextId])
.then((rows, fields) => {
if(rows[0].length == 1){
db.promise().query('UPDATE note SET shared = 0 WHERE id = ?', [rows[0][0]['id']])
}
})
resolve(true)
})
})
}
//text is the current text for the note that will be compared to the text in the database
Note.getDiffText = (userId, noteId, usersCurrentText, lastUpdated) => {
return new Promise((resolve, reject) => {
Note.get(userId, noteId)
.then(noteObject => {
if(!noteObject.text || !usersCurrentText || noteObject.encrypted == 1){
return resolve(null)
}
let oldText = noteObject.text.replace(/(\r\n|\n|\r)/gm,"")
let newText = usersCurrentText.replace(/(\r\n|\n|\r)/gm,"")
if(noteObject.updated == lastUpdated){
// console.log('No note diff')
return resolve(null)
}
if(noteObject.updated > lastUpdated){
newText = noteObject.text.replace(/(\r\n|\n|\r)/gm,"")
oldText = usersCurrentText.replace(/(\r\n|\n|\r)/gm,"")
}
const dmp = new DiffMatchPatch.diff_match_patch()
const diff = dmp.diff_main(oldText, newText)
dmp.diff_cleanupSemantic(diff)
const patch_list = dmp.patch_make(oldText, newText, diff);
const patch_text = dmp.patch_toText(patch_list);
//Patch text - shows a list of changes
var patches = dmp.patch_fromText(patch_text);
// console.log(patch_text)
//results[1] - contains diagnostic data for patch apply, its possible it can fail
var results = dmp.patch_apply(patches, oldText);
//Compile return data for front end
const returnData = {
updatedText: results[0],
diffs: results[1].length, //Only use length for now
updated: Math.max(noteObject.updated,lastUpdated) //Return most recently updated date
}
//Final change in notes
// console.log(returnData)
resolve(returnData)
})
})
}
Note.get = (userId, noteId, password = '') => {
return new Promise((resolve, reject) => {
db.promise()
.query(`
SELECT
note_raw_text.title,
note_raw_text.text,
note_raw_text.salt,
note_raw_text.password_hint,
note_raw_text.updated as updated,
note_raw_text.decrypt_attempts_count,
note_raw_text.last_decrypted_date,
note.created,
note.pinned,
note.archived,
note.color,
note.encrypted,
count(distinct attachment.id) as attachment_count,
note.note_raw_text_id as rawTextId,
shareUser.username as shareUsername
FROM note
JOIN note_raw_text ON (note_raw_text.id = note.note_raw_text_id)
LEFT JOIN attachment ON (note.id = attachment.note_id)
LEFT JOIN user as shareUser ON (note.share_user_id = shareUser.id)
WHERE note.user_id = ? AND note.id = ? LIMIT 1`, [userId, noteId])
.then((rows, fields) => {
const nowTime = Math.round((+new Date)/1000)
let noteLockedOut = false
let noteData = rows[0][0]
const rawTextId = noteData['rawTextId']
noteData.decrypted = true
//If this is not and encrypted note, pass decrypted true, skip encryption stuff
if(noteData.encrypted == 1){
noteData.decrypted = false
}
//
//Rate Limiting
//
//Check if note is exceeding decrypt attempt limit
if(noteData.encrypted == 1){
const timeSinceLastUnlock = nowTime - noteData.last_decrypted_date
//To many attempts in less than 5 minutes, note is locked
if(noteData.decrypt_attempts_count > 3 && timeSinceLastUnlock < 300){
noteLockedOut = true
}
//its been 5 minutes, reset attempt count
if(noteData.decrypt_attempts_count > 0 && timeSinceLastUnlock > 300){
noteLockedOut = false
noteData.decrypt_attempts_count = 0
db.promise().query('UPDATE note_raw_text SET last_decrypted_date = ?, decrypt_attempts_count = 0 WHERE id = ?', [nowTime, rawTextId ])
}
}
//Note is encrypted, lets try and decipher it with the given password
if(password.length > 3 && noteData.encrypted == 1 && !noteLockedOut){
const decipheredText = cs.decrypt(password, noteData.salt, noteData.text)
//Text was decrypted, return decrypted text
if(decipheredText !== null){
noteData.decrypted = true
noteData.text = decipheredText
//Save last decrypted date, reset decrypt atempts
db.promise().query('UPDATE note_raw_text SET last_decrypted_date = ?, decrypt_attempts_count = 0 WHERE id = ?', [nowTime, rawTextId ])
}
//Text was not deciphered, delete object, never return cipher text
if(decipheredText === null){
noteData.text = '' //Never return cipher text
noteData.decryptFail = true
noteData.decrypt_attempts_count++ //Update display for user
//Update decrypt attempts
db.promise().query('UPDATE note_raw_text SET decrypt_attempts_count = decrypt_attempts_count +1 WHERE id = ?', [rawTextId ])
}
}
db.promise().query(`UPDATE note SET opened = ? WHERE (id = ?)`, [nowTime, noteId])
//Return note data
delete noteData.salt //remove salt from return data
noteData.lockedOut = noteLockedOut
resolve(noteData)
})
.catch(console.log)
})
}
//Public note share action -> may not be used
Note.getShared = (noteId) => {
return new Promise((resolve, reject) => {
db.promise()
.query('SELECT text, color FROM note WHERE id = ? AND shared = 1 LIMIT 1', [noteId])
.then((rows, fields) => {
//Return note data
resolve(rows[0][0])
})
.catch(console.log)
})
}
// Searches text index, returns nothing if there is no search query
Note.solrQuery = (userId, searchQuery, searchTags) => {
return new Promise((resolve, reject) => {
if(searchQuery.length == 0){
resolve(null)
} else {
//Number of characters before and after search word
const front = 20
const tail = 150
db.promise()
.query(`
SELECT
note_id,
substring(
text,
IF(LOCATE(?, text) > ${tail}, LOCATE(?, text) - ${front}, 1),
${tail} + LENGTH(?) + ${front}
) as snippet
FROM note_text_index
WHERE user_id = ?
AND MATCH(text)
AGAINST(? IN NATURAL LANGUAGE MODE)
LIMIT 1000
;
`, [searchQuery, searchQuery, searchQuery, userId, searchQuery])
.then((rows, fields) => {
let results = []
let snippets = {}
rows[0].forEach(item => {
let noteId = parseInt(item['note_id'])
//Setup array of ids to use for query
results.push( noteId )
//Get text snippet and highlight the key word
snippets[noteId] = item['snippet'].replace(new RegExp(searchQuery,"ig"), '<em>'+searchQuery+'</em>');
//.replace(searchQuery,'<em>'+searchQuery+'</em>')
})
resolve({ 'ids':results, 'snippets':snippets })
})
}
})
}
Note.search = (userId, searchQuery, searchTags, fastFilters) => {
return new Promise((resolve, reject) => {
//Define return data objects
let returnData = {
'notes':[],
'tags':[]
}
Note.solrQuery(userId, searchQuery, searchTags).then( (textSearchResults) => {
//Pull out search results from previous query
let textSearchIds = []
let highlights = {}
let returnTagResults = false
let searchAllNotes = false
if(textSearchResults != null){
textSearchIds = textSearchResults['ids']
highlights = textSearchResults['snippets']
}
//No results, return empty data
if(textSearchIds.length == 0 && searchQuery.length > 0){
return resolve(returnData)
}
// Base of the query, modified with fastFilters
// Add to query for character counts -> CHAR_LENGTH(note.text) as chars
let searchParams = [userId]
let noteSearchQuery = `
SELECT note.id,
SUBSTRING(note_raw_text.text, 1, 500) as text,
note_raw_text.title as title,
note_raw_text.updated as updated,
opened,
color,
count(distinct note_tag.id) as tag_count,
count(distinct attachment.id) as attachment_count,
note.pinned,
note.archived,
note.encrypted,
GROUP_CONCAT(DISTINCT tag.text) as tags,
GROUP_CONCAT(DISTINCT attachment.file_location) as thumbs,
shareUser.username as shareUsername,
note.shared
FROM note
JOIN note_raw_text ON (note_raw_text.id = note.note_raw_text_id)
LEFT JOIN note_tag ON (note.id = note_tag.note_id)
LEFT JOIN tag ON (tag.id = note_tag.tag_id)
LEFT JOIN attachment ON (note.id = attachment.note_id AND attachment.visible = 1)
LEFT JOIN user as shareUser ON (note.share_user_id = shareUser.id)
WHERE note.user_id = ?
`
//Show shared notes
if(fastFilters.onlyShowSharedNotes == 1){
//share_user_id means your shared them, a note with a shared user id filled in means it was shared
noteSearchQuery += ` AND share_user_id IS NOT NULL OR (note.shared = 2 AND note.user_id = ?)`
searchParams.push(userId)
//Show notes shared with you
} else {
noteSearchQuery += ' AND note.share_user_id IS NULL'
}
//If text search returned results, limit search to those ids
if(textSearchIds.length > 0){
searchParams.push(textSearchIds)
noteSearchQuery += ' AND note.id IN (?)'
searchAllNotes = true
}
if(fastFilters.noteIdSet && fastFilters.noteIdSet.length > 0){
searchParams.push(fastFilters.noteIdSet)
noteSearchQuery += ' AND note.id IN (?)'
searchAllNotes = true
}
//Encrypted Note
if(fastFilters.onlyShowEncrypted == 1){
noteSearchQuery += ' AND encrypted = 1'
searchAllNotes = true
}
//If tags are passed, use those tags in search
if(searchTags.length > 0){
searchParams.push(searchTags)
noteSearchQuery += ' AND note_tag.tag_id IN (?)'
}
//Show archived notes, only if fast filter is set, default to not archived
if(searchAllNotes == false){
if(fastFilters.onlyArchived == 1){
noteSearchQuery += ' AND note.archived = 1' //Show Archived
} else {
noteSearchQuery += ' AND note.archived = 0' //Exclude archived
}
}
//Finish up note query
noteSearchQuery += ' GROUP BY note.id'
//Only show notes with Tags
if(fastFilters.withTags == 1){
returnTagResults = true
noteSearchQuery += ' HAVING tag_count > 0'
}
//Only show notes with links
if(fastFilters.withLinks == 1){
returnTagResults = true
noteSearchQuery += ' HAVING attachment_count > 0'
}
//Only show archived notes
if(fastFilters.onlyArchived == 1){
returnTagResults = true
noteSearchQuery += ' HAVING note.archived = 1'
}
//
// Always prioritize pinned notes in searches.
//Default Sort, order by last updated
let defaultOrderBy = ' ORDER BY note.pinned DESC, updated DESC, note.created DESC, note.opened DESC, id DESC'
//Order by Last Created Date
if(fastFilters.lastCreated == 1){
defaultOrderBy = ' ORDER BY note.pinned DESC, note.created DESC, updated DESC, note.opened DESC, id DESC'
}
//Order by last Opened Date
if(fastFilters.lastOpened == 1){
defaultOrderBy = ' ORDER BY note.pinned DESC, opened DESC, updated DESC, note.created DESC, id DESC'
}
//Append Order by to query
noteSearchQuery += defaultOrderBy
//Manage limit params if set
if(fastFilters.limitSize > 0 || fastFilters.limitOffset > 0){
const limitSize = parseInt(fastFilters.limitSize, 10) || 10 //Use int or default to 10
const limitOffset = parseInt(fastFilters.limitOffset, 10) || 0 //Either parse int, or use zero
// console.log(` LIMIT ${limitOffset}, ${limitSize}`)
noteSearchQuery += ` LIMIT ${limitOffset}, ${limitSize}`
}
// console.log('------------- Final Query --------------')
// console.log(noteSearchQuery)
// console.log('------------- ----------- --------------')
db.promise()
.query(noteSearchQuery, searchParams)
.then((noteRows, noteFields) => {
//Push all notes
returnData['notes'] = noteRows[0]
//pull out all note ids so we can fetch all tags for those notes
let noteIds = []
returnData['notes'].forEach(note => {
//Grab note ID for finding tags
noteIds.push(note.id)
if(note.text == null){ note.text = '' }
if(note.encrypted == 1){ note.text = '' }
//Deduce note title
const textData = ProcessText.deduceNoteTitle(note.title, note.text)
// console.log(textData)
note.title = textData.title
note.subtext = textData.sub
note.titleLength = textData.titleLength
note.subtextLength = textData.subtextLength
note.note_highlights = []
note.attachment_highlights = []
note.tag_highlights = []
//Limit number of attachment thumbs to 4
if(note.thumbs){
//Convert comma delimited string to array
let thumbArray = note.thumbs.split(',').reverse()
//Limit array to 4 or size of array
thumbArray.length = Math.min(thumbArray.length, 4)
note.thumbs = thumbArray
}
//Push in search highlights
if(highlights && highlights[note.id]){
note['note_highlights'] = [highlights[note.id]]
}
//Clear out note.text before sending it to front end, its being used in title and subtext
delete note.text
})
//If no notes are returned, there are no tags, return empty
if(noteIds.length == 0){
return resolve(returnData)
}
//Return all notes, tags are not being searched
// if tags are being searched, continue
// if notes are being filtered, return tags
if(searchTags.length == 0 && returnTagResults == false){
return resolve(returnData)
}
//Only show tags of selected notes
db.promise()
.query(`SELECT tag.id, tag.text, count(tag.id) as usages FROM note_tag
JOIN tag ON (tag.id = note_tag.tag_id)
WHERE note_tag.user_id = ?
AND note_id IN (?)
GROUP BY tag.id
ORDER BY usages DESC;`,[userId, noteIds])
.then((tagRows, tagFields) => {
returnData['tags'] = tagRows[0]
resolve(returnData)
})
.catch(console.log)
})
.catch(console.log)
})
.catch(console.log)
})
}