From df073b0e4df839e386eddfab1871bb819d449121 Mon Sep 17 00:00:00 2001 From: Max G Date: Wed, 6 May 2020 07:10:27 +0000 Subject: [PATCH] Fully Encrypted notes Beta * Encrypts all notes going to the database * Creates encrypted snippets for loading note title cards * Creates an encrypted search index when note is changed * Migrates users to encrypted notes on login * Creates new encrypted master keys for newly logged in users --- client/src/App.vue | 3 +- client/src/components/NoteInputPanel.vue | 94 +--- .../src/components/NoteTitleDisplayCard.vue | 4 +- client/src/components/SearchInput.vue | 2 +- client/src/pages/LoginPage.vue | 4 +- client/src/pages/NotesPage.vue | 28 +- client/src/stores/mainStore.js | 2 +- server/helpers/Auth.js | 4 +- server/helpers/CryptoString.js | 4 + server/index.js | 17 +- server/models/Note.js | 479 +++++++++++++----- server/models/User.js | 132 ++++- server/routes/noteController.js | 26 +- server/routes/userController.js | 23 +- 14 files changed, 553 insertions(+), 269 deletions(-) diff --git a/client/src/App.vue b/client/src/App.vue index 7cf7798..2c3b6e7 100644 --- a/client/src/App.vue +++ b/client/src/App.vue @@ -30,6 +30,7 @@ export default { //Puts token into state on page load let token = localStorage.getItem('loginToken') let username = localStorage.getItem('username') + let masterKey = localStorage.getItem('masterKey') // const socket = io({ path:'/socket' }); const socket = this.$io @@ -50,7 +51,7 @@ export default { //Put user data into global store on load if(token){ - this.$store.commit('setLoginToken', {token, username}) + this.$store.commit('setLoginToken', {token, username, masterKey}) } }, diff --git a/client/src/components/NoteInputPanel.vue b/client/src/components/NoteInputPanel.vue index 17aecfe..6f61905 100644 --- a/client/src/components/NoteInputPanel.vue +++ b/client/src/components/NoteInputPanel.vue @@ -86,17 +86,6 @@
- -
- -
- - -
- -
-
@@ -125,8 +114,9 @@
-
-
{{loadingMessage}}
+
+ Decrypting Note & + {{loadingMessage}}
@@ -267,55 +257,6 @@
- -
-

Note Decrypted

-
Lock Note
-
- -
- -
- -

Password protect this Note

-

Password protection will prevent anyone from reading the text of this note, unless they enter the correct password.

-

Only the note text is protected. Title, tags, and files are not encrypted and remain visible without a password.

-

The password you select will only be used for this note. You can use the same password on multiple notes. The note will be encrypted using the password entered. A longer password will be more secure.

-

Warning. There is no way to recover a lost password.

- -
- -
-
-
- - -
-
- - -
-
- - -
- -
-
- Passwords do not match -
-
-
-
- Protect! -
-
-
-
- -
- -
{ @@ -1255,6 +1200,11 @@ this.save().then( result => { + //If note was modified, trigger reindex on close + if(this.modified){ + axios.post('/api/note/reindex') + } + this.sizeDown = true //This timeout allows animation to play before closing setTimeout(() => { @@ -1488,11 +1438,23 @@ } .loading-note { position: absolute; - top: 0; - left: 0; - right: 0; - bottom: 0; + top: 20%; + left: 20%; + right: 20%; + bottom: 20%; + background: transparent; + color: #5e6268;; + font-size: 1.3em; } + .loading-text { + margin: 0; + position: absolute; + top: 50%; + left: 50%; + margin-right: -50%; + transform: translate(-50%, -50%); + } + /* One note open, in the middle of the screen */ .master-note-edit.position-0 { left: 50%; diff --git a/client/src/components/NoteTitleDisplayCard.vue b/client/src/components/NoteTitleDisplayCard.vue index 7bda5fa..0558df4 100644 --- a/client/src/components/NoteTitleDisplayCard.vue +++ b/client/src/components/NoteTitleDisplayCard.vue @@ -68,7 +68,7 @@
-
- +
diff --git a/client/src/pages/LoginPage.vue b/client/src/pages/LoginPage.vue index 8a2be0c..f53cbbf 100644 --- a/client/src/pages/LoginPage.vue +++ b/client/src/pages/LoginPage.vue @@ -80,12 +80,14 @@ const token = response.data.token const username = response.data.username + const masterKey = response.data.masterKey - vm.$store.commit('setLoginToken', {token, username}) + vm.$store.commit('setLoginToken', {token, username, masterKey}) //Redirect user to notes section after login vm.$router.push('/notes') } else { + // this.password = '' this.$bus.$emit('notification', 'Incorrect Username or Password') vm.$store.commit('destroyLoginToken') } diff --git a/client/src/pages/NotesPage.vue b/client/src/pages/NotesPage.vue index bcbce2a..49b930f 100644 --- a/client/src/pages/NotesPage.vue +++ b/client/src/pages/NotesPage.vue @@ -64,7 +64,7 @@

Notes with Tags

Archived Notes

Shared Notes

-

Password Protected Notes

+

Password Protected - No longer supported

@@ -154,14 +154,15 @@ highlights: [], searchDebounce: null, fastFilters: {}, - working: false, + //Load up notes in batches - firstLoadBatchSize: 30, //First set of rapidly loaded notes - batchSize: 100, //Size of batch loaded when user scrolls through current batch + firstLoadBatchSize: 10, //First set of rapidly loaded notes + batchSize: 25, //Size of batch loaded when user scrolls through current batch batchOffset: 0, //Tracks the current batch that has been loaded loadingBatchTimeout: null, //Limit how quickly batches can be loaded loadingInProgress: false, fetchTags: false, + scrollLoadEnabled: true, //Clear button is not visible showClear: false, @@ -237,9 +238,9 @@ return } }) - }) - + }) }) + this.$bus.$on('update_fast_filters', newFilter => { this.fastFilters = newFilter //Fast filters always return all the results and tags @@ -254,7 +255,8 @@ this.search(true, this.batchSize) .then( () => { - this.searchAttachments() + console.log('Search attachments disabled for now') + // this.searchAttachments() return this.fetchUserTags() }) @@ -275,6 +277,7 @@ const id = this.$route.params.id this.openNote(id) } + window.addEventListener('scroll', this.onScroll) //Close notes when back button is pressed @@ -411,7 +414,7 @@ const percentageDown = Math.round( (bottomOfWindow/offsetHeight)*100 ) //If greater than 80 of the way down the page, load the next batch - if(percentageDown >= 80){ + if(percentageDown >= 65 && this.scrollLoadEnabled){ this.search(false, this.batchSize, true) } @@ -455,7 +458,7 @@ }, visibiltyChangeAction(event){ - //@TODO - set a timeout on this like 2 minutes or just dont do shit and update it via socket.io + //@TODO - phase this out, update it via socket.io //If user leaves page then returns to page, reload the first batch if(this.lastVisibilityState == 'hidden' && document.visibilityState == 'visible'){ //Load initial batch, then tags, then other batch @@ -589,12 +592,18 @@ //Perform search - or die this.loadingInProgress = true + console.time('Fetch TitleCard Batch '+notesInNextLoad) axios.post('/api/note/search', postData) .then(response => { + console.timeEnd('Fetch TitleCard Batch '+notesInNextLoad) + //Save the number of notes just loaded this.batchOffset += response.data.notes.length + //Enable or disable scroll loading + this.scrollLoadEnabled = response.data.notes.length > 0 + //Mush the two new sets of data together (set will be empty is reset is on) if(response.data.tags.length > 0){ this.commonTags = response.data.tags @@ -666,6 +675,7 @@ }, reset(){ this.showClear = false + this.scrollLoadEnabled = true this.searchTerm = '' this.searchTags = [] this.fastFilters = {} diff --git a/client/src/stores/mainStore.js b/client/src/stores/mainStore.js index 57ea220..c6ded44 100644 --- a/client/src/stores/mainStore.js +++ b/client/src/stores/mainStore.js @@ -87,6 +87,7 @@ export default new Vuex.Store({ })(navigator.userAgent||navigator.vendor||window.opera, state); }, toggleNoteSettingsPane(state){ + state.isNoteSettingsOpen = !state.isNoteSettingsOpen }, setSocketIoSocket(state, socket){ @@ -103,7 +104,6 @@ export default new Vuex.Store({ // console.log(key + ' -- ' + totalsObject[key]) // }) } - }, getters: { getUsername: state => { diff --git a/server/helpers/Auth.js b/server/helpers/Auth.js index af10fd8..656cc53 100644 --- a/server/helpers/Auth.js +++ b/server/helpers/Auth.js @@ -4,8 +4,8 @@ let Auth = {} const tokenSecretKey = process.env.JSON_KEY -Auth.createToken = (userId) => { - const signedData = {'id': userId, 'date':Date.now()} +Auth.createToken = (userId, masterKey) => { + const signedData = {'id':userId, 'date':Date.now(), 'masterKey':masterKey} const token = jwt.sign(signedData, tokenSecretKey) return token } diff --git a/server/helpers/CryptoString.js b/server/helpers/CryptoString.js index 8b57798..ea50503 100644 --- a/server/helpers/CryptoString.js +++ b/server/helpers/CryptoString.js @@ -69,6 +69,10 @@ CryptoString.createSalt = () => { return crypto.randomBytes(SALT_BYTE_SIZE).toString('base64') } +CryptoString.createSmallSalt = () => { + + return crypto.randomBytes(20).toString('base64') +} CryptoString.hash = (hashString) => { diff --git a/server/index.js b/server/index.js index aad7412..ed87cf2 100644 --- a/server/index.js +++ b/server/index.js @@ -124,6 +124,7 @@ app.use(function(req, res, next){ Auth.decodeToken(token) .then(userData => { req.headers.userId = userData.id //Update headers for the rest of the application + req.headers.masterKey = userData.masterKey next() }).catch(error => { @@ -135,17 +136,11 @@ app.use(function(req, res, next){ } }) -// Testing Area -// let att = require('@models/Attachment') -// let testUrl = 'https://dba.stackexchange.com/questions/23908/how-to-search-a-mysql-database-with-encrypted-fields' -// testUrl = 'https://www.solidscribe.com/#/' -// console.log('About to scrape: ', testUrl) -// att.processUrl(61, 3213, testUrl) -// .then(results => { -// console.log('Scrape happened') -// }) -// -// + +// Test Area +// -> right here +// Test Area + //Test app.get(prefix, (req, res) => res.send('The api is running')) diff --git a/server/models/Note.js b/server/models/Note.js index 0aa452f..2d542e0 100644 --- a/server/models/Note.js +++ b/server/models/Note.js @@ -16,105 +16,95 @@ let Note = module.exports = {} const gm = require('gm') -// -------------- - -Note.migrateNoteTextToNewTable = () => { +//User doesn't have an encrypted note set. Encrypt all notes +Note.encryptEveryNote = (userId, masterKey) => { return new Promise((resolve, reject) => { - db.promise() - .query('SELECT id, text FROM note WHERE note_raw_text_id IS NULL') + + //Select all the user notes + db.promise().query(` + SELECT * FROM note + JOIN note_raw_text ON (note_raw_text.id = note.note_raw_text_id) + WHERE salt IS NULL AND user_id = ? AND encrypted = 0 AND shared = 0`, [userId]) .then((rows, fields) => { - rows[0].forEach( ({id, text}) => { - db.promise() - .query('INSERT INTO note_raw_text (text) VALUES (?)', [text]) - .then((rows, fields) => { + let foundNotes = rows[0] + console.log('Encrypting user notes ',rows[0].length) - db.promise() - .query(`UPDATE note SET note_raw_text_id = ? WHERE (id = ?)`, [rows[0].insertId, id]) - .then((rows, fields) => { + // return resolve(true) - return 'Nice' - }) - }) + let allTheUpdates = [] + let timeoutAdder = 0 + foundNotes.forEach(note => { + timeoutAdder += 100 + const newUpdate = new Promise((resolve, reject) => { + setTimeout(() => { + console.log('Encrypting Note ', note.id) - }) + const created = Math.round((+new Date)/1000) + const salt = cs.createSmallSalt() - resolve('Its probably running... :-D') - }) - }) -} + const noteText = note.text + const noteTitle = note.title -Note.fixAttachmentThumbnails = () => { - const filePath = '../staticFiles/' - db.promise() - .query(`SELECT * FROM attachment WHERE file_location NOT LIKE "%.%"`) - .then( (rows, fields) => { + const snippet = JSON.stringify([noteTitle, noteText.substring(0, 500)]) + const noteSnippet = cs.encrypt(masterKey, salt, snippet) - rows[0].forEach(line => { + const textObject = JSON.stringify([noteTitle, noteText]) + const encryptedText = cs.encrypt(masterKey, salt, textObject) - const rawFilename = line['file_location'] - const goodFileName = rawFilename+'.jpg' - - //Rename file to have jpg extension, create thumbnail, update database - fs.rename(filePath+rawFilename, filePath+goodFileName, (err) => { - - db.promise() - .query(`UPDATE attachment SET file_location = ? WHERE id = ?`,[goodFileName, line['id'] ]) - .then( (rows, fields) => { - gm(filePath+goodFileName) - .resize(550) //Resize to width of 550 px - .quality(75) //compression level 0 - 100 (best) - .write(filePath + 'thumb_'+goodFileName, function (err) { - console.log('Done for -> ', goodFileName) - }) + db.promise() + .query('UPDATE note_raw_text SET title = ?, text = ?, snippet = ?, salt = ? WHERE id = ?', + [null, encryptedText, noteSnippet, salt, note.note_raw_text_id]) + .then(() => { + resolve(true) }) + }, timeoutAdder) }) - + allTheUpdates.push(newUpdate) }) - }) -} -Note.stressTest = () => { - return new Promise((resolve, reject) => { - db.promise() - .query(` + Promise.all(allTheUpdates).then(done => { + + console.log('Indexing first 100') + return Note.reindex(userId, masterKey) + + }).then(results => { + + console.log('Done') + resolve(true) + }) + + }) + + + + - SELECT text FROM note; - `) - .then((rows, fields) => { - console.log() - - rows[0].forEach(item => { - - Note.create(68, item['text']) - }) - - resolve(true) - }) - .catch(console.log) }) } -// -------------- - -Note.create = (userId, noteTitle, noteText, quickNote = 0, ) => { +Note.create = (userId, noteTitle, noteText, masterKey) => { return new Promise((resolve, reject) => { if(userId == null || userId < 10){ reject('User Id required to create note') } const created = Math.round((+new Date)/1000) + const salt = cs.createSmallSalt() + + const textObject = JSON.stringify([noteTitle, noteText]) + const encryptedText = cs.encrypt(masterKey, salt, textObject) db.promise() - .query(`INSERT INTO note_raw_text (text, title, updated) VALUE (?, ?, ?)`, [noteText, noteTitle, created]) + .query(`INSERT INTO note_raw_text (text, salt, updated) VALUE (?, ?, ?)`, [encryptedText, salt, created]) .then( (rows, fields) => { const rawTextId = rows[0].insertId return db.promise() .query('INSERT INTO note (user_id, note_raw_text_id, created, quick_note) VALUES (?,?,?,?)', - [userId, rawTextId, created, quickNote]) + [userId, rawTextId, created, 0]) }) .then((rows, fields) => { // Indexing is done on save @@ -124,9 +114,193 @@ Note.create = (userId, noteTitle, noteText, quickNote = 0, ) => { }) } -Note.reindex = (userId, noteId) => { +Note.reindex = (userId, masterKey) => { return new Promise((resolve, reject) => { + if(!masterKey || masterKey.length == 0){ + return reject('Master key needed for reindex') + } + + let notIndexedNoteIds = [] + let searchIndex = null + let searchIndexSalt = null + let foundNotes = null + + //First check if we have any notes to index + db.promise().query(` + SELECT note.id, text, salt FROM note + JOIN note_raw_text ON note.note_raw_text_id = note_raw_text.id + WHERE indexed = 0 AND encrypted = 0 AND salt IS NOT NULL + AND user_id = ? LIMIT 100`, [userId]) + .then((rows, fields) => { + + //Halt execution if there are no new notes + foundNotes = rows[0] + if(foundNotes.length == 0){ + throw new Error('No new notes to index') + } + + //Select search index, if it doesn't exist, create it + return db.promise().query(`SELECT * FROM user_encrypted_search_index WHERE user_id = ? LIMIT 1`, [userId]) + + }) + .then((rows, fields) => { + + if(rows[0].length == 0){ + + console.log('Creating a new index') + //Create search index entry, return an object + searchIndexSalt = cs.createSmallSalt() + + //Select all user notes to recreate index + return db.promise().query(` + SELECT note.id, text, salt FROM note + JOIN note_raw_text ON note.note_raw_text_id = note_raw_text.id + WHERE encrypted = 0 AND user_id = ?`, [userId]) + .then((rows, fields) => { + + foundNotes = rows[0] + return db.promise().query("INSERT INTO user_encrypted_search_index (`user_id`, `salt`) VALUES (?,?)", [userId, searchIndexSalt]) + }) + .then((rows, fields) => { + //return a fresh search index + return new Promise((resolve, reject) => { resolve('{}') }) + }) + + + } else { + + const row = rows[0][0] + searchIndexSalt = row.salt + + //Decrypt search index and continue. + let decipheredSearchIndex = '{}' + if(row.index && row.index.length > 0){ + //Decrypt json, do not parse json yet, we want raw text + decipheredSearchIndex = cs.decrypt(masterKey, searchIndexSalt, row.index) + } + + return new Promise((resolve, reject) => { resolve( decipheredSearchIndex ) }) + } + + }) + .then(rawSearchIndex => { + + searchIndex = rawSearchIndex + + //Remove all instances of IDs from text + foundNotes.forEach(note => { + + notIndexedNoteIds.push(note.id) + + //Remove every instance of note id + const removeId = new RegExp(note.id,"gm") + const removeDoubles = new RegExp(',,',"g") + // const removeTrail = new RegExp(',]',"g") + + + searchIndex = searchIndex + .replace(removeId, '') + .replace(removeDoubles, ',') + .replace(/,]/g, ']') + .replace(/\[\,/g, '[') //search [, + }) + + searchIndex = JSON.parse(searchIndex) + + //Remove unused words, this may not be needed and it increases overhead + Object.keys(searchIndex).forEach(word => { + if(searchIndex[word].length == 0){ + delete searchIndex[word] + } + }) + + //Process text of each note and add it to the index + let reindexQueue = [] + let reindexTimer = 0 + foundNotes.forEach(note => { + + reindexTimer += 50 + let reindexPromise = new Promise((resolve, reject) => { + setTimeout(() => { + + if(masterKey == null || note.salt == null){ + console.log('Error indexing note', note.id) + return resolve(true) + } + + const noteHtml = cs.decrypt(masterKey, note.salt, note.text) + + const rawText = + ProcessText.removeHtml(noteHtml) //Remove HTML + .toLowerCase() + .replace(/style=".*?"/g,'') //Remove inline styles + .replace (/&#{0,1}[a-z0-9]+;/ig, '') //remove HTML entities + .replace(/[^A-Za-z0-9]/g, ' ') //Convert all to a-z only + .replace(/ +(?= )/g,'') //Remove double spaces + + rawText.split(' ').forEach(word => { + + //Skip small words + if(word.length <= 2){ return } + + if(Array.isArray( searchIndex[word] )){ + if(searchIndex[word].indexOf( note.id ) == -1){ + searchIndex[word].push( note.id ) + } + } else { + searchIndex[word] = [ note.id ] + } + }) + + return resolve(true) + + }, reindexTimer) + }) + + reindexQueue.push(reindexPromise) + + }) + + return Promise.all(reindexQueue) + }) + .then(rawSearchIndex => { + + console.log('All notes indexed') + + const created = Math.round((+new Date)/1000) + const jsonSearchIndex = JSON.stringify(searchIndex) + const encryptedJsonIndex = cs.encrypt(masterKey, searchIndexSalt, jsonSearchIndex) + + return db.promise().query("UPDATE user_encrypted_search_index SET `index` = ?, `last_update` = ? WHERE (`user_id` = ?) LIMIT 1", + [encryptedJsonIndex, created, userId]) + .then((rows, fields) => { + + return db.promise().query('UPDATE note SET `indexed` = 1 WHERE (`id` IN (?))', [notIndexedNoteIds]) + + }) + .then((rows, fields) => { + + console.log('Indexd Note Count: ' + rows[0]['affectedRows']) + resolve(true) + + }) + + }).catch(error => { + console.log('Reindex Error') + console.log(error) + }) + + + + + + + //Find all note Ids that need to be reindexed + + // return resolve(true) + return + Note.get(userId, noteId) .then(note => { @@ -163,15 +337,9 @@ Note.reindex = (userId, noteId) => { }) } -Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, password = '', passwordHint = '') => { +Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, password = '', passwordHint = '', masterKey) => { return new Promise((resolve, reject) => { - //Prevent note loss if it saves with empty text - //if(ProcessText.removeHtml(noteText) == ''){ - // console.log('Not saving empty note') - // resolve(false) - //} - const now = Math.round((+new Date)/1000) db.promise() @@ -183,11 +351,7 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, const textId = rows[0][0]['note_raw_text_id'] let salt = rows[0][0]['salt'] - - //If password is removed, remove salt. generate a new one next time its encrypted - if(password.length == 0){ - salt = null - } + let noteSnippet = '' //If a password is set, create a salt if(password.length > 3 && !salt){ @@ -206,11 +370,20 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, // // @TODO - Do note save data if encryption goes wrong, do some validation // + } else { + + //Create encrypted snippet + const snippet = JSON.stringify([noteTitle, noteText.substring(0, 500)]) + noteSnippet = cs.encrypt(masterKey, salt, snippet) + + //Encrypt note text + const textObject = JSON.stringify([noteTitle, noteText]) + noteText = cs.encrypt(masterKey, salt, textObject) } //Update Note text return db.promise() - .query('UPDATE note_raw_text SET text = ?, title = ?, updated = ?, salt = ? WHERE id = ?', [noteText, noteTitle, now, salt, textId]) + .query('UPDATE note_raw_text SET text = ?, snippet = ? ,updated = ?, salt = ? WHERE id = ?', [noteText, noteSnippet, now, salt, textId]) }) .then( (rows, fields) => { @@ -218,14 +391,14 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, //Update other note attributes return db.promise() - .query('UPDATE note SET pinned = ?, archived = ?, color = ?, encrypted = ? WHERE id = ? AND user_id = ? LIMIT 1', + .query('UPDATE note SET pinned = ?, archived = ?, color = ?, encrypted = ?, indexed = 0 WHERE id = ? AND user_id = ? LIMIT 1', [pinned, archived, color, encrypted, noteId, userId]) }) .then((rows, fields) => { //Async solr note reindex - Note.reindex(userId, noteId) + // Note.reindex(userId, noteId) //Async attachment reindex Attachment.scanTextForWebsites(io, userId, noteId, noteText) @@ -392,12 +565,16 @@ Note.getDiffText = (userId, noteId, usersCurrentText, lastUpdated) => { } -Note.get = (userId, noteId, password = '') => { +Note.get = (userId, noteId, password = '', masterKey) => { return new Promise((resolve, reject) => { + + if(!masterKey || masterKey.length == 0){ + return reject('Get note called without master key') + } + db.promise() .query(` SELECT - note_raw_text.title, note_raw_text.text, note_raw_text.salt, note_raw_text.password_hint, @@ -482,6 +659,17 @@ Note.get = (userId, noteId, password = '') => { db.promise().query('UPDATE note_raw_text SET decrypt_attempts_count = decrypt_attempts_count +1 WHERE id = ?', [rawTextId ]) } } + if(noteData.encrypted == 0 && noteData.salt && noteData.salt.length > 0){ + //Normal Encrypted note + const decipheredText = cs.decrypt(masterKey, noteData.salt, noteData.text) + if(decipheredText == null){ + throw new Error('Unable to decropt note text') + } + //Parse title and text from encrypted data and update object + const textObject = JSON.parse(decipheredText) + noteData.title = textObject[0] + noteData.text = textObject[1] + } db.promise().query(`UPDATE note SET opened = ? WHERE (id = ?)`, [nowTime, noteId]) @@ -511,56 +699,75 @@ Note.getShared = (noteId) => { } // Searches text index, returns nothing if there is no search query -Note.solrQuery = (userId, searchQuery, searchTags) => { +Note.solrQuery = (userId, searchQuery, searchTags, masterKey) => { return new Promise((resolve, reject) => { if(searchQuery.length == 0){ resolve(null) } else { - //Number of characters before and after search word - const front = 20 - const tail = 150 + if(!masterKey || masterKey == null){ + console.log('Attempting to search wiouth key') + return resolve(null) + } - db.promise() - .query(` - - SELECT - note_id, - substring( - text, - IF(LOCATE(?, text) > ${tail}, LOCATE(?, text) - ${front}, 1), - ${tail} + LENGTH(?) + ${front} - ) as snippet - FROM note_text_index - WHERE user_id = ? - AND MATCH(text) - AGAINST(? IN NATURAL LANGUAGE MODE) - LIMIT 1000 - ; - - `, [searchQuery, searchQuery, searchQuery, userId, searchQuery]) + //Search the search index + db.promise().query(`SELECT * FROM user_encrypted_search_index WHERE user_id = ? LIMIT 1`, [userId]) .then((rows, fields) => { - let results = [] - let snippets = {} - rows[0].forEach(item => { - let noteId = parseInt(item['note_id']) - //Setup array of ids to use for query - results.push( noteId ) - //Get text snippet and highlight the key word - snippets[noteId] = item['snippet'].replace(new RegExp(searchQuery,"ig"), ''+searchQuery+''); - //.replace(searchQuery,''+searchQuery+'') - }) + if(rows[0].length == 1){ + + //Lookup, decrypt and parse search index + const row = rows[0][0] + const decipheredSearchIndex = cs.decrypt(masterKey, row.salt, row.index) + const searchIndex = JSON.parse(decipheredSearchIndex) + + //Clean up search word + const word = searchQuery.toLowerCase().replace(/[^a-z0-9]/g, '') + + let noteIds = [] + let partials = [] + Object.keys(searchIndex).forEach(wordIndex => { + if( wordIndex.indexOf(word) != -1 && wordIndex != word){ + partials.push(wordIndex) + noteIds.push(...searchIndex[wordIndex]) + } + }) + + const exactArray = searchIndex[word] ? searchIndex[word] : [] + + let searchData = { + 'word':word, + 'exact': exactArray, + 'partials': partials, + 'partial': [...new Set(noteIds) ], + } + + //Remove exact matches from partials set if there is overlap + if(searchData['exact'].length > 0 && searchData['partial'].length > 0){ + searchData['partial'] = searchData['partial'] + .filter( ( el ) => !searchData['exact'].includes( el ) ) + } + + searchData['ids'] = searchData['exact'].concat(searchData['partial']) + searchData['total'] = searchData['ids'].length + + console.log(searchData['total']) + + return resolve({ 'ids':searchData['ids'] }) + + + } else { + return resolve(null) + } - resolve({ 'ids':results, 'snippets':snippets }) }) } }) } -Note.search = (userId, searchQuery, searchTags, fastFilters) => { +Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => { return new Promise((resolve, reject) => { //Define return data objects let returnData = { @@ -568,17 +775,16 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => { 'tags':[] } - Note.solrQuery(userId, searchQuery, searchTags).then( (textSearchResults) => { + Note.solrQuery(userId, searchQuery, searchTags, masterKey).then( (textSearchResults) => { //Pull out search results from previous query let textSearchIds = [] - let highlights = {} let returnTagResults = false let searchAllNotes = false if(textSearchResults != null){ textSearchIds = textSearchResults['ids'] - highlights = textSearchResults['snippets'] + // highlights = textSearchResults['snippets'] } //No results, return empty data @@ -588,11 +794,14 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => { // Base of the query, modified with fastFilters // Add to query for character counts -> CHAR_LENGTH(note.text) as chars + + //SUBSTRING(note_raw_text.text, 1, 500) as text, let searchParams = [userId] let noteSearchQuery = ` - SELECT note.id, - SUBSTRING(note_raw_text.text, 1, 500) as text, + SELECT note.id, note_raw_text.title as title, + note_raw_text.snippet as snippet, + note_raw_text.salt as salt, note_raw_text.updated as updated, opened, color, @@ -725,17 +934,24 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => { //Grab note ID for finding tags noteIds.push(note.id) - if(note.text == null){ note.text = '' } - if(note.encrypted == 1){ note.text = '' } + if(note.encrypted == 1){ + note.text = '' + } + //Decrypt note text + if(note.snippet && note.salt){ + const decipheredText = cs.decrypt(masterKey, note.salt, note.snippet) + const textObject = JSON.parse(decipheredText) + if(textObject != null && textObject.length == 2){ + note.title = textObject[0] + note.text = textObject[1] + } + } //Deduce note title const textData = ProcessText.deduceNoteTitle(note.title, note.text) - // console.log(textData) note.title = textData.title note.subtext = textData.sub - note.titleLength = textData.titleLength - note.subtextLength = textData.subtextLength note.note_highlights = [] note.attachment_highlights = [] @@ -750,13 +966,10 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => { note.thumbs = thumbArray } - //Push in search highlights - if(highlights && highlights[note.id]){ - note['note_highlights'] = [highlights[note.id]] - } - //Clear out note.text before sending it to front end, its being used in title and subtext delete note.text + delete note.snippet + delete note.salt }) //If no notes are returned, there are no tags, return empty diff --git a/server/models/User.js b/server/models/User.js index 19ab820..80759d1 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -1,7 +1,10 @@ var crypto = require('crypto') -let db = require('@config/database') -let Auth = require('@helpers/Auth') +const Note = require('@models/Note') + +const db = require('@config/database') +const Auth = require('@helpers/Auth') +const cs = require('@helpers/CryptoString') let User = module.exports = {} @@ -22,26 +25,32 @@ User.login = (username, password) => { //User not found, create a new account with set data if(rows[0].length == 0){ User.create(lowerName, password) - .then(loginToken => { - resolve(loginToken) + .then( ({token, userId}) => { + return resolve({ token, userId }) }) - return } - //hash the password and check for a match - const salt = new Buffer(lookedUpUser.salt, 'binary') - crypto.pbkdf2(password, salt, lookedUpUser.iterations, 512, 'sha512', function(err, delivered_key){ - if(delivered_key.toString('hex') === lookedUpUser.password){ + if(lookedUpUser && lookedUpUser.salt){ + //hash the password and check for a match + const salt = new Buffer(lookedUpUser.salt, 'binary') + crypto.pbkdf2(password, salt, lookedUpUser.iterations, 512, 'sha512', function(err, delivered_key){ + if(delivered_key.toString('hex') === lookedUpUser.password){ - //Passback a json web token - const token = Auth.createToken(lookedUpUser.id) - resolve(token) + User.generateMasterKey(lookedUpUser.id, password) + .then( result => User.getMasterKey(lookedUpUser.id, password)) + .then(masterKey => { - } else { + //Passback a json web token + const token = Auth.createToken(lookedUpUser.id, masterKey) + resolve({ token: token, userId:lookedUpUser.id }) + }) - reject('Password does not match database') - } - }) + } else { + + reject('Password does not match database') + } + }) + } }) .catch(console.log) @@ -93,9 +102,15 @@ User.create = (username, password) => { if(rows[0].affectedRows == 1){ - const newUserId = rows[0].insertId - const loginToken = Auth.createToken(newUserId) - resolve(loginToken) + const userId = rows[0].insertId + + User.generateMasterKey(userId, password) + .then( result => User.getMasterKey(userId, password)) + .then(masterKey => { + + const token = Auth.createToken(userId, masterKey) + return resolve({token, userId}) + }) } else { //Emit Error to user @@ -166,5 +181,84 @@ User.getCounts = (userId) => { resolve(countTotals) }) + }) +} + +User.generateMasterKey = (userId, password) => { + return new Promise((resolve, reject) => { + + if(!userId || !password){ + reject('Need userId and password to generate key') + } + + db.promise() + .query('SELECT count(id) as total FROM user_key WHERE user_id = ?', [userId]) + .then((rows, fields) => { + + //Entry already exists, you good. + if(rows[0][0]['total'] > 0){ + return resolve(true) + // throw new Error('User Encryption key already exists') + } else { + // Generate user key, its big and random + const masterPassword = cs.createSmallSalt() + console.log('Generating new key for user', userId) + + //Generate a salt because it wants it + const salt = cs.createSmallSalt() + + // Encrypt master password + const encryptedMasterPassword = cs.encrypt(password, salt, masterPassword) + + const created = Math.round((+new Date)/1000) + + db.promise() + .query( + 'INSERT INTO user_key (`user_id`, `salt`, `key`, `created`) VALUES (?, ?, ?, ?);', + [userId, salt, encryptedMasterPassword, created] + ) + .then((rows, fields)=>{ + return Note.encryptEveryNote(userId, masterPassword) + }) + .then(results => { + return new Promise((resolve, reject) => { resolve(true) }) + }) + } + + }) + .then((rows, fields) => { + return resolve(true) + }) + .catch(error => { + console.log('Create Master Password Error') + console.log(error) + }) + + }) +} + +User.getMasterKey = (userId, password) => { + + if(!userId || !password){ + reject('Need userId and password to fetch key') + } + + return new Promise((resolve, reject) => { + + db.promise().query('SELECT * FROM user_key WHERE user_id = ? LIMIT 1', [userId]) + .then((rows, fields) => { + + const row = rows[0][0] + + const masterKey = cs.decrypt(password, row['salt'], row['key']) + + if(masterKey == null){ + return reject('Unable to decrypt key') + } + + return resolve(masterKey) + + }) + }) } \ No newline at end of file diff --git a/server/routes/noteController.js b/server/routes/noteController.js index 19e9cbd..1b126b8 100644 --- a/server/routes/noteController.js +++ b/server/routes/noteController.js @@ -5,15 +5,13 @@ let Notes = require('@models/Note'); let ShareNote = require('@models/ShareNote'); let userId = null -let socket = null +let masterKey = null // middleware that is specific to this router router.use(function setUserId (req, res, next) { if(req.headers.userId){ userId = req.headers.userId - } - if(req.headers.socket){ - // socket = req. + masterKey = req.headers.masterKey } next() @@ -23,11 +21,8 @@ router.use(function setUserId (req, res, next) { // Note actions // router.post('/get', function (req, res) { - // req.io.emit('welcome_homie', 'Welcome, dont poop from excitement') - Notes.get(userId, req.body.noteId, req.body.password) + Notes.get(userId, req.body.noteId, req.body.password, masterKey) .then( data => { - //Join room when user opens note - // req.io.join('note_room') res.send(data) }) }) @@ -38,17 +33,17 @@ router.post('/delete', function (req, res) { }) router.post('/create', function (req, res) { - Notes.create(userId, req.body.title, req.body.text) + Notes.create(userId, req.body.title, req.body.text, masterKey) .then( id => res.send({id}) ) }) router.post('/update', function (req, res) { - Notes.update(req.io, userId, req.body.noteId, req.body.text, req.body.title, req.body.color, req.body.pinned, req.body.archived, req.body.password, req.body.hint) + Notes.update(req.io, userId, req.body.noteId, req.body.text, req.body.title, req.body.color, req.body.pinned, req.body.archived, req.body.password, req.body.hint, masterKey) .then( id => res.send({id}) ) }) router.post('/search', function (req, res) { - Notes.search(userId, req.body.searchQuery, req.body.searchTags, req.body.fastFilters) + Notes.search(userId, req.body.searchQuery, req.body.searchTags, req.body.fastFilters, masterKey) .then( notesAndTags => { res.send(notesAndTags) }) @@ -62,6 +57,14 @@ router.post('/difftext', function (req, res) { }) }) +router.post('/reindex', function (req, res) { + Notes.reindex(userId, masterKey) + .then( data => { + res.send(data) + }) +}) + + // // Update single note attributes // @@ -116,5 +119,4 @@ router.get('/reindex5yu43prchuj903mrc', function (req, res) { }) - module.exports = router \ No newline at end of file diff --git a/server/routes/userController.js b/server/routes/userController.js index 14b66ee..05ed924 100644 --- a/server/routes/userController.js +++ b/server/routes/userController.js @@ -2,6 +2,7 @@ var express = require('express') var router = express.Router() let User = require('@models/User'); +const cs = require('@helpers/CryptoString') // middleware that is specific to this router router.use(function timeLog (req, res, next) { @@ -31,19 +32,19 @@ router.post('/login', function (req, res) { } User.login(username, password) - .then(function(loginToken){ + .then( ({token, userId}) => { - //Return json web token to user - returnData['success'] = true - returnData['token'] = loginToken - returnData['username'] = username + returnData['username'] = username + returnData['token'] = token + returnData['success'] = true - res.send(returnData) - }) - .catch(e => { - console.log(e) - res.send(returnData) - }) + res.send(returnData) + return + }) + .catch(e => { + console.log(e) + res.send(returnData) + }) }) // fetch counts of users notes