Max/SolidScribe
1
0 Fork 0
Code Issues 24 Pull Requests Releases Wiki Activity

* Removed arrows from notification

Browse Source 
* Added trash can function
* Tweaked status text to always be the same
* Removed some open second note code
* Edior always focuses on text now
* Added some extra loading note messages
* Notes are now removed from search index when deleted
* Lots more things happen and update in real time on multiple machines
* Shared notes can be reverted
* WAY more tests
* Note Categories are much more reliable
* Lots of code is much cleaner
This commit is contained in:
Max G committed 2020-05-18 07:45:35 +00:00
1 parent e87e8513bc
commit 5096e74a60
18 files changed
+644 -368
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/models/Note.js
+243 -152
View File
@@ -14,81 +14,152 @@ const crypto = require('crypto')
const cs = require('@helpers/CryptoString')
const rp = require('request-promise');
const fs = require('fs')
const gm = require('gm')
Note.test = (userId, masterKey) => {
Note.test = (userId, masterKey, printResults) => {
return new Promise((resolve, reject) => {
let testNoteId = 0
Note.create(null, userId, '','', masterKey)
let testNoteId = 0
let testNoteId2 = 0
let sharedNoteId = 0 //ID of note shared with user
const shareUserId = 61
Note.create(userId, 'Random Note','With Random Text dogs', masterKey)
.then( newNoteId => {
if(printResults) console.log('Test: Created Note -> ', newNoteId)
return Note.create(userId, 'Yo, note','a second test note cheese mate', masterKey)
})
.then( newNoteId => {
if(printResults) console.log('Test: Created Note -> ', newNoteId)
testNoteId2 = newNoteId
//Create a blank note to test updating note and reindexing it
return Note.create(userId, '','', masterKey)
})
.then(newNoteId => {
console.log('Test: Create Note - Pass')
if(printResults) console.log('Test: Created Note -> ', newNoteId)
testNoteId = newNoteId
return Note.update
(null, userId, testNoteId, 'Note text', 'Test Note beans Title', 0, 0, 0, 'hash', masterKey)
(userId, testNoteId, 'Note text', 'Test Note beans Title', 0, 0, 0, 'hash', masterKey)
})
.then(() => {
console.log('Test: Update Note - Pass')
return Note.get(userId, testNoteId, masterKey)
})
.then(updatedText => {
console.log('Test: Open Updated Note - Pass')
const shareUserId = 61
return ShareNote.migrateNoteToShared(userId, testNoteId, shareUserId, masterKey)
})
.then(shareResults => {
console.log('Test: Set Note To Shared - Pass')
if(printResults) console.log('Test: Update Note '+testNoteId+' - Pass')
return Note.get(userId, testNoteId, masterKey)
})
.then(() => {
console.log('Test: Open Shared Note - Pass')
return Note.update
(null, userId, testNoteId, 'Shared Update', 'Test Note beans Title', 0, 0, 0, 'hash', masterKey)
})
.then(() => {
if(printResults) console.log('Test: Open Updated Note - Pass')
console.log('Test: Update Shared Note - Pass')
return Note.reindex(userId, masterKey)
})
.then( reindexResults => {
console.log(`Test: Reindex Notes - ${reindexResults?'Pass':'Fail'}`)
})
.then(() => {
if(printResults) console.log('Test: Reindex normal note - Pass')
return Note.encryptedIndexSearch(userId, 'beans', null, masterKey)
})
.then(textSearchResults => {
if(textSearchResults['ids'] && textSearchResults['ids'].length >= 1){
console.log('Test: Search Index - Pass')
if(printResults) console.log('Test: Normal Note Search Index - Pass')
} else { console.log('Test: Search Index - Fail') }
return Note.delete(userId, testNoteId)
return ShareNote.migrateNoteToShared(userId, testNoteId, shareUserId, masterKey)
})
.then(({success, shareUserId, sharedUserNoteId}) => {
if(printResults) console.log('Test: Set Note To Shared - Pass')
sharedNoteId = sharedUserNoteId
return Note.get(userId, testNoteId, masterKey)
})
.then(() => {
if(printResults) console.log('Test: Open Shared Note - Pass')
return Note.update
(userId, testNoteId, 'Shared Update', 'Test Note yarnsmarf Title', 0, 0, 0, 'hash', masterKey)
})
.then(() => {
if(printResults) console.log('Test: Update Shared Note - Pass')
return Note.reindex(userId, masterKey)
})
.then( reindexResults => {
if(printResults) console.log(`Test: Reindex Notes - ${reindexResults?'Pass':'Fail'}`)
return Note.encryptedIndexSearch(userId, 'yarnsmarf', null, masterKey)
})
.then(textSearchResults => {
if(textSearchResults['ids'] && textSearchResults['ids'].length >= 1){
if(printResults) console.log('Test: Search Index - Pass')
} else { console.log('Test: Search Index - Fail') }
return Note.delete(userId, testNoteId, masterKey)
})
.then(results => {
console.log('Test: Delete Note - Pass')
if(printResults) console.log('Test: Delete Note - Pass')
return resolve('Test: Complete')
return Note.encryptedIndexSearch(userId, 'yarnsmarf', null, masterKey)
})
.then(textSearchResults => {
if(textSearchResults['ids'] && textSearchResults['ids'].length == 0){
if(printResults) console.log('Test: Search Deleted Note Text - Pass')
} else { console.log('Test: Search Deleted Note Text - Fail') }
return Note.delete(shareUserId, sharedNoteId)
})
.then(() => {
if(printResults) console.log('Test: Delete Shared Note - Pass')
return ShareNote.migrateNoteToShared(userId, testNoteId2, shareUserId, masterKey)
})
.then(({success, shareUserId, sharedUserNoteId}) => {
if(printResults) console.log('Test: Created Another New Shared Note - pass')
return ShareNote.removeUserFromShared(userId, testNoteId2, sharedUserNoteId, masterKey)
})
.then(() => {
if(printResults) console.log('Test: Unshared New Shared Note -> convert back to normal note - pass')
return Note.get(userId, testNoteId2, masterKey)
})
.then(() => {
if(printResults) console.log('Test: Decrypt unshared note - pass')
let User = require('@models/User')
return User.deleteUser(userId, '1')
})
.then(results => {
if(printResults) console.log('Test: Delete User Account - Pass')
return resolve('Test: Complete ---')
})
})
@@ -103,7 +174,7 @@ Note.encryptEveryNote = (userId, masterKey) => {
db.promise().query(`
SELECT * FROM note
JOIN note_raw_text ON (note_raw_text.id = note.note_raw_text_id)
WHERE salt IS NULL AND user_id = ? AND encrypted = 0 AND shared = 0`, [userId])
WHERE salt IS NULL AND user_id = ? AND shared = 0`, [userId])
.then((rows, fields) => {
let foundNotes = rows[0]
@@ -164,7 +235,7 @@ Note.encryptEveryNote = (userId, masterKey) => {
}
//Returns insertedId of new note
Note.create = (io, userId, noteTitle = '', noteText = '', masterKey) => {
Note.create = (userId, noteTitle = '', noteText = '', masterKey) => {
return new Promise((resolve, reject) => {
if(userId == null || userId < 10){ reject('User Id required to create note') }
@@ -186,13 +257,13 @@ Note.create = (io, userId, noteTitle = '', noteText = '', masterKey) => {
const rawTextId = rows[0].insertId
return db.promise()
.query('INSERT INTO note (user_id, note_raw_text_id, created, quick_note, snippet, snippet_salt) VALUES (?,?,?,?,?,?)',
.query('INSERT INTO note (user_id, note_raw_text_id, created, quick_note, snippet, snippet_salt, indexed) VALUES (?,?,?,?,?,?,0)',
[userId, rawTextId, created, 0, snippet, snippetSalt])
})
.then((rows, fields) => {
if(io){
io.to(userId).emit('new_note_created', rows[0].insertId)
if(SocketIo){
SocketIo.to(userId).emit('new_note_created', rows[0].insertId)
}
// Indexing is done on save
@@ -205,7 +276,8 @@ Note.create = (io, userId, noteTitle = '', noteText = '', masterKey) => {
// Called when a note is close
// Will attempt to reindex all notes that are flagged in database as not indexed
// Limit to 100 notes per batch
Note.reindex = (userId, masterKey) => {
// removeId = array of Ids to remove, [122,223]
Note.reindex = (userId, masterKey, removeId = null) => {
return new Promise((resolve, reject) => {
if(!masterKey || masterKey.length == 0){
@@ -216,17 +288,37 @@ Note.reindex = (userId, masterKey) => {
let searchIndex = null
let searchIndexSalt = null
let foundNotes = null
let userPrivateKey = null
//First check if we have any notes to index
db.promise().query(`
SELECT note.id, text, salt FROM note
JOIN note_raw_text ON note.note_raw_text_id = note_raw_text.id
WHERE indexed = 0 AND encrypted = 0 AND salt IS NOT NULL
AND user_id = ? LIMIT 100`, [userId])
let User = require('@models/User')
User.generateKeypair(userId, masterKey)
.then(({publicKey, privateKey}) => {
userPrivateKey = privateKey
//First check if we have any notes to index
return db.promise().query(`
SELECT note.id, text, salt, encrypted_share_password_key FROM note
JOIN note_raw_text ON note.note_raw_text_id = note_raw_text.id
WHERE indexed = 0 AND salt IS NOT NULL
AND user_id = ? LIMIT 100`, [userId])
})
.then((rows, fields) => {
//Halt execution if there are no new notes
foundNotes = rows[0]
//Remove ID from index but don't reindex text
if(removeId != null){
removeId.forEach(removeId => {
foundNotes.push({
id:removeId,
text:'',
salt: null,
})
})
}
if(foundNotes.length == 0){
throw new Error('No new notes to index')
}
@@ -244,9 +336,9 @@ Note.reindex = (userId, masterKey) => {
//Select all user notes to recreate index
return db.promise().query(`
SELECT note.id, text, salt FROM note
SELECT note.id, text, salt, encrypted_share_password_key FROM note
JOIN note_raw_text ON note.note_raw_text_id = note_raw_text.id
WHERE encrypted = 0 AND user_id = ?`, [userId])
WHERE user_id = ?`, [userId])
.then((rows, fields) => {
foundNotes = rows[0]
@@ -276,6 +368,10 @@ Note.reindex = (userId, masterKey) => {
})
.then(rawSearchIndex => {
if(rawSearchIndex == null){
throw new Error('Search Index Not Found/Decrypted')
}
searchIndex = rawSearchIndex
//Remove all instances of IDs from text
@@ -288,7 +384,6 @@ Note.reindex = (userId, masterKey) => {
const removeDoubles = new RegExp(',,',"g")
// const removeTrail = new RegExp(',]',"g")
searchIndex = searchIndex
.replace(removeId, '')
.replace(removeDoubles, ',')
@@ -319,7 +414,17 @@ Note.reindex = (userId, masterKey) => {
return resolve(true)
}
const noteHtml = cs.decrypt(masterKey, note.salt, note.text)
let currentNoteKey = masterKey
//Decrypt shared key if it exists
const encryptedShareKey = note.encrypted_share_password_key
if(encryptedShareKey != null){
currentNoteKey = crypto.privateDecrypt(userPrivateKey,
Buffer.from(encryptedShareKey, 'base64') )
}
//Decrypt text with proper key
const noteHtml = cs.decrypt(currentNoteKey, note.salt, note.text)
const rawText =
ProcessText.removeHtml(noteHtml) //Remove HTML
@@ -356,8 +461,6 @@ Note.reindex = (userId, masterKey) => {
})
.then(rawSearchIndex => {
// console.log('All notes indexed')
const created = Math.round((+new Date)/1000)
const jsonSearchIndex = JSON.stringify(searchIndex)
const encryptedJsonIndex = cs.encrypt(masterKey, searchIndexSalt, jsonSearchIndex)
@@ -377,58 +480,14 @@ Note.reindex = (userId, masterKey) => {
})
}).catch(error => {
console.log('Reindex Error')
console.log('Reindex Error:')
console.log(error)
})
//Find all note Ids that need to be reindexed
// return resolve(true)
return
Note.get(userId, noteId)
.then(note => {
let noteText = note.text
if(note.encrypted == 1){
noteText = '' //Don't put note text in encrypted notes
}
//
// Update Solr index
//
Tags.string(userId, noteId)
.then(tagString => {
const fullText = note.title + ' ' + ProcessText.removeHtml(noteText) +' '+ tagString
db.promise()
.query(`
INSERT INTO note_text_index (note_id, user_id, text)
VALUES (?,?,?)
ON DUPLICATE KEY UPDATE text = ?
`, [noteId, userId, fullText, fullText])
.then((rows, fields) => {
resolve(true)
})
.catch(console.log)
})
})
})
}
// Returns updated note text
Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, hash, masterKey) => {
Note.update = (userId, noteId, noteText, noteTitle, color, pinned, archived, hash, masterKey) => {
return new Promise((resolve, reject) => {
const now = Math.round((+new Date)/1000)
@@ -483,6 +542,8 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived,
//Re-encrypt for other user
const updatedSnippet = cs.encrypt(masterKey, otherNote.snippet_salt, snippet)
db.promise().query('UPDATE note SET snippet = ? WHERE id = ?', [updatedSnippet, otherNote.id])
SocketIo.to(otherNote['user_id']).emit('new_note_text_saved', {'noteId':otherNote.id, hash})
}
})
@@ -501,12 +562,12 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived,
})
.then((rows, fields) => {
if(io){
io.to(userId).emit('new_note_text_saved', {noteId, hash})
if(SocketIo){
SocketIo.to(userId).emit('new_note_text_saved', {noteId, hash})
}
//Async attachment reindex
Attachment.scanTextForWebsites(io, userId, noteId, noteText)
Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
//Send back updated response
resolve(rows[0])
@@ -519,12 +580,14 @@ Note.setPinned = (userId, noteId, pinnedBoolean) => {
return new Promise((resolve, reject) => {
const pinned = pinnedBoolean ? 1:0
const now = Math.round((+new Date)/1000)
//Update other note attributes
return db.promise()
.query('UPDATE note SET pinned = ? WHERE id = ? AND user_id = ? LIMIT 1',
[pinned, noteId, userId])
.query('UPDATE note JOIN note_raw_text ON note_raw_text_id = note_raw_text.id SET pinned = ?, updated = ? WHERE note.id = ? AND user_id = ?',
[pinned, now, noteId, userId])
.then((rows, fields) => {
SocketIo.to(userId).emit('note_attribute_modified', noteId)
resolve(true)
})
})
@@ -534,12 +597,31 @@ Note.setArchived = (userId, noteId, archivedBoolead) => {
return new Promise((resolve, reject) => {
const archived = archivedBoolead ? 1:0
const now = Math.round((+new Date)/1000)
//Update other note attributes
return db.promise()
.query('UPDATE note SET archived = ? WHERE id = ? AND user_id = ? LIMIT 1',
[archived, noteId, userId])
.query('UPDATE note JOIN note_raw_text ON note_raw_text_id = note_raw_text.id SET archived = ?, updated = ? WHERE note.id = ? AND user_id = ?',
[archived, now, noteId, userId])
.then((rows, fields) => {
SocketIo.to(userId).emit('note_attribute_modified', noteId)
resolve(true)
})
})
}
Note.setTrashed = (userId, noteId, trashedBoolean) => {
return new Promise((resolve, reject) => {
const trashed = trashedBoolean ? 1:0
const now = Math.round((+new Date)/1000)
//Update other note attributes
return db.promise()
.query('UPDATE note JOIN note_raw_text ON note_raw_text_id = note_raw_text.id SET trashed = ?, updated = ? WHERE note.id = ? AND user_id = ?',
[trashed, now, noteId, userId])
.then((rows, fields) => {
SocketIo.to(userId).emit('note_attribute_modified', noteId)
resolve(true)
})
})
@@ -548,7 +630,7 @@ Note.setArchived = (userId, noteId, archivedBoolead) => {
//
// Delete a note and all its remaining parts
//
Note.delete = (userId, noteId) => {
Note.delete = (userId, noteId, masterKey = null) => {
return new Promise((resolve, reject) => {
//
@@ -591,10 +673,10 @@ Note.delete = (userId, noteId) => {
return db.promise()
.query('DELETE FROM note WHERE note.id = ? AND note.user_id = ?', [noteId, userId])
})
.then((rows, fields) => {
// Delete search index
.then( results => {
// Delete hidden attachments for this note (files for attachment are already gone)
return db.promise()
.query('DELETE FROM note_text_index WHERE note_text_index.note_id = ? AND note_text_index.user_id = ?', [noteId,userId])
.query('DELETE FROM attachment WHERE visible = 0 AND note_id = ? AND user_id = ?', [noteId, userId])
})
.then((rows, fields) => {
// delete tags
@@ -603,7 +685,11 @@ Note.delete = (userId, noteId) => {
})
.then((rows, fields) => {
//IF there are nots with a matching raw text id, we want to under their share status
})
.then((rows, fields) => {
//IF there are notes with a matching raw text id, we want to update their share status
db.promise().query('SELECT id FROM note WHERE note_raw_text_id = ?',[rawTextId])
.then((rows, fields) => {
if(rows[0].length == 1){
@@ -611,7 +697,15 @@ Note.delete = (userId, noteId) => {
}
})
resolve(true)
if(masterKey){
//Remove note ID from index
Note.reindex(userId, masterKey, [noteId])
.then(results => {
return resolve(true)
})
} else {
return resolve(true)
}
})
})
}
@@ -698,6 +792,7 @@ Note.get = (userId, noteId, masterKey) => {
note.created,
note.pinned,
note.archived,
note.trashed,
note.color,
note.encrypted_share_password_key,
count(distinct attachment.id) as attachment_count,
@@ -889,7 +984,7 @@ Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
count(distinct attachment.id) as attachment_count,
note.pinned,
note.archived,
note.encrypted,
note.trashed,
GROUP_CONCAT(DISTINCT tag.text) as tags,
GROUP_CONCAT(DISTINCT attachment.file_location) as thumbs,
shareUser.username as shareUsername,
@@ -903,16 +998,6 @@ Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
LEFT JOIN user as shareUser ON (note.share_user_id = shareUser.id)
WHERE note.user_id = ?
`
//Show shared notes
if(fastFilters.onlyShowSharedNotes == 1){
//share_user_id means your shared them, a note with a shared user id filled in means it was shared
noteSearchQuery += ` AND share_user_id IS NOT NULL OR (note.shared = 2 AND note.user_id = ?)`
searchParams.push(userId)
//Show notes shared with you
} else {
noteSearchQuery += ' AND note.share_user_id IS NULL'
}
//If text search returned results, limit search to those ids
if(textSearchIds.length > 0){
@@ -921,18 +1006,13 @@ Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
searchAllNotes = true
}
//If Specific ID's are being searched, search ALL notes
if(fastFilters.noteIdSet && fastFilters.noteIdSet.length > 0){
searchParams.push(fastFilters.noteIdSet)
noteSearchQuery += ' AND note.id IN (?)'
searchAllNotes = true
}
//Encrypted Note
if(fastFilters.onlyShowEncrypted == 1){
noteSearchQuery += ' AND encrypted = 1'
searchAllNotes = true
}
//If tags are passed, use those tags in search
if(searchTags.length > 0){
searchParams.push(searchTags)
@@ -941,31 +1021,48 @@ Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
//Show archived notes, only if fast filter is set, default to not archived
if(searchAllNotes == false){
if(fastFilters.onlyArchived == 1){
noteSearchQuery += ' AND note.archived = 1' //Show Archived
} else {
noteSearchQuery += ' AND note.archived = 0' //Exclude archived
if(fastFilters.notesHome == 1){
noteSearchQuery += ' AND note.archived = 0 AND note.trashed = 0 AND note.share_user_id IS NULL'
}
if(fastFilters.onlyShowSharedNotes == 1){
//share_user_id means your shared them, a note with a shared user id filled in means it was shared
noteSearchQuery += ` AND share_user_id IS NOT NULL OR (note.shared = 2 AND note.user_id = ?) AND note.trashed = 0`
searchParams.push(userId)
//Show notes shared with you
}
if(fastFilters.onlyArchived == 1){
noteSearchQuery += ' AND note.archived = 1 AND note.trashed = 0' //Show Archived
}
if(fastFilters.onlyShowTrashed == 1){
noteSearchQuery += ' AND note.trashed = 1' //Show Exclude
}
}
//Finish up note query
noteSearchQuery += ' GROUP BY note.id'
//Only show notes with Tags
if(fastFilters.withTags == 1){
returnTagResults = true
noteSearchQuery += ' HAVING tag_count > 0'
}
//Only show notes with links
if(fastFilters.withLinks == 1){
returnTagResults = true
noteSearchQuery += ' HAVING attachment_count > 0'
}
//Only show archived notes
if(fastFilters.onlyArchived == 1){
returnTagResults = true
noteSearchQuery += ' HAVING note.archived = 1'
}
// if(fastFilters.onlyArchived == 1){
// noteSearchQuery += ' HAVING note.archived = 1'
// }
// //Only show trashed notes
// if(fastFilters.onlyShowTrashed == 1){
// noteSearchQuery += ' HAVING note.trashed = 1'
// }
//
// Always prioritize pinned notes in searches.
@@ -992,8 +1089,6 @@ Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
const limitSize = parseInt(fastFilters.limitSize, 10) || 10 //Use int or default to 10
const limitOffset = parseInt(fastFilters.limitOffset, 10) || 0 //Either parse int, or use zero
// console.log(` LIMIT ${limitOffset}, ${limitSize}`)
noteSearchQuery += ` LIMIT ${limitOffset}, ${limitSize}`
}
@@ -1005,19 +1100,14 @@ Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
.query(noteSearchQuery, searchParams)
.then((noteRows, noteFields) => {
//Current note key may change, default to master key
let currentNoteKey = masterKey
//Push all notes
returnData['notes'] = noteRows[0]
//pull out all note ids so we can fetch all tags for those notes
let noteIds = []
returnData['notes'].forEach(note => {
//Grab note ID for finding tags
noteIds.push(note.id)
//Current note key may change, default to master key
let currentNoteKey = masterKey
//Shared notes use encrypted key - decrypt key then decrypt note
const encryptedShareKey = note.encrypted_share_password_key
@@ -1060,6 +1150,7 @@ Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
//Clear out note.text before sending it to front end, its being used in title and subtext
delete note.snippet
delete note.salt
delete note.encrypted_share_password_key
})