Max/SolidScribe
1
0
Fork 0
Code Issues 24 Pull Requests Releases Wiki Activity

Fully Encrypted notes Beta

Browse Source 
* Encrypts all notes going to the database
* Creates encrypted snippets for loading note title cards
* Creates an encrypted search index when note is changed
* Migrates users to encrypted notes on login
* Creates new encrypted master keys for newly logged in users
This commit is contained in:
Max G
2020-05-06 07:10:27 +00:00
parent a545ced98f
commit df073b0e4d
14 changed files with 553 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
client/src/App.vue
View File

@@ -30,6 +30,7 @@ export default {
//Puts token into state on page load //Puts token into state on page load
let token = localStorage.getItem('loginToken') let token = localStorage.getItem('loginToken')
let username = localStorage.getItem('username') let username = localStorage.getItem('username')
let masterKey = localStorage.getItem('masterKey')
// const socket = io({ path:'/socket' }); // const socket = io({ path:'/socket' });
const socket = this.$io const socket = this.$io
@@ -50,7 +51,7 @@ export default {
//Put user data into global store on load //Put user data into global store on load
if(token){ if(token){
this.$store.commit('setLoginToken', {token, username}) this.$store.commit('setLoginToken', {token, username, masterKey})
} }
}, },

94
client/src/components/NoteInputPanel.vue
View File

@@ -86,17 +86,6 @@
<div class="edit-divide"></div> <div class="edit-divide"></div>
<!-- protect -->
<div class="edit-button" v-if="!isEncrypted"
v-on:click="$router.push(`/notes/open/${noteid}/menu/passwordprotect`)" data-tooltip="Password Protect" data-position="bottom center" data-inverted>
<i class="shield alternate icon"></i>
</div>
<!-- data-tooltip="Remove Password Protection" -->
<div class="edit-button" v-if="isEncrypted && isDecrypted" v-on:click="disableEncryption()" data-tooltip="Close" data-position="bottom center" data-inverted>
<i class="unlock icon"></i>
</div>
<div class="edit-button" v-on:click="onToggleArchived()" :data-tooltip="archived == 1?'Move to main list':'Move to Archive'" data-position="bottom center" data-inverted> <div class="edit-button" v-on:click="onToggleArchived()" :data-tooltip="archived == 1?'Move to main list':'Move to Archive'" data-position="bottom center" data-inverted>
<span v-if="archived == 1"><i class="green archive icon"></i></span> <span v-if="archived == 1"><i class="green archive icon"></i></span>
<span v-if="archived != 1"><i class="archive icon"></i></span> <span v-if="archived != 1"><i class="archive icon"></i></span>
@@ -125,8 +114,9 @@
<!-- Loading indicator --> <!-- Loading indicator -->
<div v-if="loading" class="loading-note"> <div v-if="loading" class="loading-note">
<div class="ui active dimmer"> <div class="loading-text">
<div class="ui text loader">{{loadingMessage}}</div> Decrypting Note &
{{loadingMessage}}
</div> </div>
</div> </div>
@@ -267,55 +257,6 @@
</div> </div>
</side-slide-menu> </side-slide-menu>
<side-slide-menu v-show="passwordprotect" v-on:close="passwordprotect = false" :fullShadow="true" name="encrypt note">
<div class="ui basic segment" v-if="isDecrypted && isEncrypted">
<p>Note Decrypted</p>
<div class="ui green button" v-on:click="lockNote">Lock Note</div>
</div>
<div v-if="!isEncrypted" class="ui basic segment">
<div class="ui top attached segment">
<h2><i class="green lock alternate icon"></i>Password protect this Note</h2>
<p>Password protection will prevent anyone from reading the text of this note, unless they enter the correct password.</p>
<p><b>Only the note text is protected. Title, tags, and files are not encrypted and remain visible without a password.</b></p>
<p>The password you select will only be used for this note. You can use the same password on multiple notes. The note will be encrypted using the password entered. A longer password will be more secure.</p>
<h4><i class="red icon exclamation triangle"></i> Warning. There is no way to recover a lost password.</h4>
</div>
<div class="ui bottom attached segment">
<div class="ui form">
<div class="field">
<label>New Password to lock this note</label>
<input :name="`randomThing-${noteid}`" :id="`yupper-${noteid}`"type="password" v-model="password" placeholder="Note Password">
</div>
<div class="field" v-if="password.length > 3">
<label>Confirm Password</label>
<input :name="`randomStuff-${noteid}`" :id="`heckye-${noteid}`"type="password" v-model="passwordConfirm" placeholder="Confirm Password">
</div>
<div class="field" v-if="password.length > 3">
<label>Password Hint - visible when unlocking note</label>
<input :name="`randomStuzz-${noteid}`" :id="`heckyo-${noteid}`"type="text" v-model="passwordHint" placeholder="Optional Password Hint" v-on:keyup.enter="enableEncryption">
</div>
<div class="field" v-if="passwordConfirm.length > 3 && password != passwordConfirm">
<div v-on:click="enableEncryption" class="ui disabled green button">
Passwords do not match
</div>
</div>
<div class="field" v-if="passwordConfirm.length > 3 && password == passwordConfirm">
<div v-on:click="enableEncryption" class="ui green button">
Protect!
</div>
</div>
</div>
</div>
</div>
</side-slide-menu>
<!-- Show side shades if user is on desktop only --> <!-- Show side shades if user is on desktop only -->
<div class="full-focus-shade shade1" <div class="full-focus-shade shade1"
:class="{ 'slide-out-left':(sizeDown == true) }" :class="{ 'slide-out-left':(sizeDown == true) }"
@@ -1206,6 +1147,10 @@
updated: this.updated updated: this.updated
} }
console.log('Focus regained with note open.')
console.log('Attempting to fix diff text. fix this. Search spleen')
return
axios.post('/api/note/difftext', postData) axios.post('/api/note/difftext', postData)
.then( ({data}) => { .then( ({data}) => {
@@ -1255,6 +1200,11 @@
this.save().then( result => { this.save().then( result => {
//If note was modified, trigger reindex on close
if(this.modified){
axios.post('/api/note/reindex')
}
this.sizeDown = true this.sizeDown = true
//This timeout allows animation to play before closing //This timeout allows animation to play before closing
setTimeout(() => { setTimeout(() => {
@@ -1488,11 +1438,23 @@
} }
.loading-note { .loading-note {
position: absolute; position: absolute;
top: 0; top: 20%;
left: 0; left: 20%;
right: 0; right: 20%;
bottom: 0; bottom: 20%;
background: transparent;
color: #5e6268;;
font-size: 1.3em;
} }
.loading-text {
margin: 0;
position: absolute;
top: 50%;
left: 50%;
margin-right: -50%;
transform: translate(-50%, -50%);
}
/* One note open, in the middle of the screen */ /* One note open, in the middle of the screen */
.master-note-edit.position-0 { .master-note-edit.position-0 {
left: 50%; left: 50%;

4
client/src/components/NoteTitleDisplayCard.vue
View File

@@ -68,7 +68,7 @@
<div class="tool-bar" @click.self="cardClicked"> <div class="tool-bar" @click.self="cardClicked">
<div class="icon-bar"> <div class="icon-bar">
<!-- <span v-if="note.pinned == 1" data-position="top left" data-tooltip="Pinned" data-inverted> <!-- <span v-if="note.pinned == 1" data-position="top left" data-tooltip="Pinned" data-inverted>
<i class="green pin icon"></i> <i class="green pin icon"></i>
</span> </span>
<span v-if="note.archived == 1" data-position="top left" data-tooltip="Archived" data-inverted> <span v-if="note.archived == 1" data-position="top left" data-tooltip="Archived" data-inverted>
@@ -80,7 +80,7 @@
<br> <br>
</span> </span>
<span data-tooltip="Edited" class="time-ago-display" :class="{ 'hover-hide':(!$store.getters.getIsUserOnMobile) }"> <span class="time-ago-display" :class="{ 'hover-hide':(!$store.getters.getIsUserOnMobile) }">
{{$helpers.timeAgo(note.updated)}} {{$helpers.timeAgo(note.updated)}}
</span> </span>

2
client/src/components/SearchInput.vue
View File

@@ -13,7 +13,7 @@
<div class="ui form" v-if="!$store.getters.getIsUserOnMobile"> <div class="ui form" v-if="!$store.getters.getIsUserOnMobile">
<!-- normal search menu --> <!-- normal search menu -->
<div class="ui left icon fluid input"> <div class="ui left icon fluid input">
<input v-model="searchTerm" @keyup="searchKeyUp" @keyup.enter="search" placeholder="Search Notes and Files" ref="searchInput"/> <input v-model="searchTerm" @keyup.enter="search" placeholder="Search Notes and Files" ref="searchInput"/>
<i class="search icon"></i> <i class="search icon"></i>
</div> </div>
</div> </div>

4
client/src/pages/LoginPage.vue
View File

@@ -80,12 +80,14 @@
const token = response.data.token const token = response.data.token
const username = response.data.username const username = response.data.username
const masterKey = response.data.masterKey
vm.$store.commit('setLoginToken', {token, username}) vm.$store.commit('setLoginToken', {token, username, masterKey})
//Redirect user to notes section after login //Redirect user to notes section after login
vm.$router.push('/notes') vm.$router.push('/notes')
} else { } else {
// this.password = ''
this.$bus.$emit('notification', 'Incorrect Username or Password') this.$bus.$emit('notification', 'Incorrect Username or Password')
vm.$store.commit('destroyLoginToken') vm.$store.commit('destroyLoginToken')
} }

26
client/src/pages/NotesPage.vue
View File

@@ -64,7 +64,7 @@
<h2 v-if="fastFilters['withTags'] == 1">Notes with Tags</h2> <h2 v-if="fastFilters['withTags'] == 1">Notes with Tags</h2>
<h2 v-if="fastFilters['onlyArchived'] == 1">Archived Notes</h2> <h2 v-if="fastFilters['onlyArchived'] == 1">Archived Notes</h2>
<h2 v-if="fastFilters['onlyShowSharedNotes'] == 1">Shared Notes</h2> <h2 v-if="fastFilters['onlyShowSharedNotes'] == 1">Shared Notes</h2>
<h2 v-if="fastFilters['onlyShowEncrypted'] == 1">Password Protected Notes</h2> <h2 v-if="fastFilters['onlyShowEncrypted'] == 1">Password Protected - No longer supported</h2>
<!-- Note title card display --> <!-- Note title card display -->
<div class="sixteen wide column"> <div class="sixteen wide column">
@@ -154,14 +154,15 @@
highlights: [], highlights: [],
searchDebounce: null, searchDebounce: null,
fastFilters: {}, fastFilters: {},
working: false,
//Load up notes in batches //Load up notes in batches
firstLoadBatchSize: 30, //First set of rapidly loaded notes firstLoadBatchSize: 10, //First set of rapidly loaded notes
batchSize: 100, //Size of batch loaded when user scrolls through current batch batchSize: 25, //Size of batch loaded when user scrolls through current batch
batchOffset: 0, //Tracks the current batch that has been loaded batchOffset: 0, //Tracks the current batch that has been loaded
loadingBatchTimeout: null, //Limit how quickly batches can be loaded loadingBatchTimeout: null, //Limit how quickly batches can be loaded
loadingInProgress: false, loadingInProgress: false,
fetchTags: false, fetchTags: false,
scrollLoadEnabled: true,
//Clear button is not visible //Clear button is not visible
showClear: false, showClear: false,
@@ -238,8 +239,8 @@
} }
}) })
}) })
}) })
this.$bus.$on('update_fast_filters', newFilter => { this.$bus.$on('update_fast_filters', newFilter => {
this.fastFilters = newFilter this.fastFilters = newFilter
//Fast filters always return all the results and tags //Fast filters always return all the results and tags
@@ -254,7 +255,8 @@
this.search(true, this.batchSize) this.search(true, this.batchSize)
.then( () => { .then( () => {
this.searchAttachments() console.log('Search attachments disabled for now')
// this.searchAttachments()
return this.fetchUserTags() return this.fetchUserTags()
}) })
@@ -275,6 +277,7 @@
const id = this.$route.params.id const id = this.$route.params.id
this.openNote(id) this.openNote(id)
} }
window.addEventListener('scroll', this.onScroll) window.addEventListener('scroll', this.onScroll)
//Close notes when back button is pressed //Close notes when back button is pressed
@@ -411,7 +414,7 @@
const percentageDown = Math.round( (bottomOfWindow/offsetHeight)*100 ) const percentageDown = Math.round( (bottomOfWindow/offsetHeight)*100 )
//If greater than 80 of the way down the page, load the next batch //If greater than 80 of the way down the page, load the next batch
if(percentageDown >= 80){ if(percentageDown >= 65 && this.scrollLoadEnabled){
this.search(false, this.batchSize, true) this.search(false, this.batchSize, true)
} }
@@ -455,7 +458,7 @@
}, },
visibiltyChangeAction(event){ visibiltyChangeAction(event){
//@TODO - set a timeout on this like 2 minutes or just dont do shit and update it via socket.io //@TODO - phase this out, update it via socket.io
//If user leaves page then returns to page, reload the first batch //If user leaves page then returns to page, reload the first batch
if(this.lastVisibilityState == 'hidden' && document.visibilityState == 'visible'){ if(this.lastVisibilityState == 'hidden' && document.visibilityState == 'visible'){
//Load initial batch, then tags, then other batch //Load initial batch, then tags, then other batch
@@ -589,12 +592,18 @@
//Perform search - or die //Perform search - or die
this.loadingInProgress = true this.loadingInProgress = true
console.time('Fetch TitleCard Batch '+notesInNextLoad)
axios.post('/api/note/search', postData) axios.post('/api/note/search', postData)
.then(response => { .then(response => {
console.timeEnd('Fetch TitleCard Batch '+notesInNextLoad)
//Save the number of notes just loaded //Save the number of notes just loaded
this.batchOffset += response.data.notes.length this.batchOffset += response.data.notes.length
//Enable or disable scroll loading
this.scrollLoadEnabled = response.data.notes.length > 0
//Mush the two new sets of data together (set will be empty is reset is on) //Mush the two new sets of data together (set will be empty is reset is on)
if(response.data.tags.length > 0){ if(response.data.tags.length > 0){
this.commonTags = response.data.tags this.commonTags = response.data.tags
@@ -666,6 +675,7 @@
}, },
reset(){ reset(){
this.showClear = false this.showClear = false
this.scrollLoadEnabled = true
this.searchTerm = '' this.searchTerm = ''
this.searchTags = [] this.searchTags = []
this.fastFilters = {} this.fastFilters = {}

2
client/src/stores/mainStore.js
View File

@@ -87,6 +87,7 @@ export default new Vuex.Store({
})(navigator.userAgent||navigator.vendor||window.opera, state); })(navigator.userAgent||navigator.vendor||window.opera, state);
}, },
toggleNoteSettingsPane(state){ toggleNoteSettingsPane(state){
state.isNoteSettingsOpen = !state.isNoteSettingsOpen state.isNoteSettingsOpen = !state.isNoteSettingsOpen
}, },
setSocketIoSocket(state, socket){ setSocketIoSocket(state, socket){
@@ -103,7 +104,6 @@ export default new Vuex.Store({
// console.log(key + ' -- ' + totalsObject[key]) // console.log(key + ' -- ' + totalsObject[key])
// }) // })
} }
}, },
getters: { getters: {
getUsername: state => { getUsername: state => {

4
server/helpers/Auth.js
View File

@@ -4,8 +4,8 @@ let Auth = {}
const tokenSecretKey = process.env.JSON_KEY const tokenSecretKey = process.env.JSON_KEY
Auth.createToken = (userId) => { Auth.createToken = (userId, masterKey) => {
const signedData = {'id': userId, 'date':Date.now()} const signedData = {'id':userId, 'date':Date.now(), 'masterKey':masterKey}
const token = jwt.sign(signedData, tokenSecretKey) const token = jwt.sign(signedData, tokenSecretKey)
return token return token
} }

4
server/helpers/CryptoString.js
View File

@@ -69,6 +69,10 @@ CryptoString.createSalt = () => {
return crypto.randomBytes(SALT_BYTE_SIZE).toString('base64') return crypto.randomBytes(SALT_BYTE_SIZE).toString('base64')
} }
CryptoString.createSmallSalt = () => {
return crypto.randomBytes(20).toString('base64')
}
CryptoString.hash = (hashString) => { CryptoString.hash = (hashString) => {

17
server/index.js
View File

@@ -124,6 +124,7 @@ app.use(function(req, res, next){
Auth.decodeToken(token) Auth.decodeToken(token)
.then(userData => { .then(userData => {
req.headers.userId = userData.id //Update headers for the rest of the application req.headers.userId = userData.id //Update headers for the rest of the application
req.headers.masterKey = userData.masterKey
next() next()
}).catch(error => { }).catch(error => {
@@ -135,17 +136,11 @@ app.use(function(req, res, next){
} }
}) })
// Testing Area
// let att = require('@models/Attachment') // Test Area
// let testUrl = 'https://dba.stackexchange.com/questions/23908/how-to-search-a-mysql-database-with-encrypted-fields' // -> right here
// testUrl = 'https://www.solidscribe.com/#/' // Test Area
// console.log('About to scrape: ', testUrl)
// att.processUrl(61, 3213, testUrl)
// .then(results => {
// console.log('Scrape happened')
// })
//
//
//Test //Test
app.get(prefix, (req, res) => res.send('The api is running')) app.get(prefix, (req, res) => res.send('The api is running'))

479
server/models/Note.js
View File

@@ -16,105 +16,95 @@ let Note = module.exports = {}
const gm = require('gm') const gm = require('gm')
// -------------- //User doesn't have an encrypted note set. Encrypt all notes
Note.encryptEveryNote = (userId, masterKey) => {
Note.migrateNoteTextToNewTable = () => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
db.promise()
.query('SELECT id, text FROM note WHERE note_raw_text_id IS NULL') //Select all the user notes
db.promise().query(`
SELECT * FROM note
JOIN note_raw_text ON (note_raw_text.id = note.note_raw_text_id)
WHERE salt IS NULL AND user_id = ? AND encrypted = 0 AND shared = 0`, [userId])
.then((rows, fields) => { .then((rows, fields) => {
rows[0].forEach( ({id, text}) => {
db.promise() let foundNotes = rows[0]
.query('INSERT INTO note_raw_text (text) VALUES (?)', [text]) console.log('Encrypting user notes ',rows[0].length)
.then((rows, fields) => {
db.promise() // return resolve(true)
.query(`UPDATE note SET note_raw_text_id = ? WHERE (id = ?)`, [rows[0].insertId, id])
.then((rows, fields) => {
return 'Nice' let allTheUpdates = []
}) let timeoutAdder = 0
}) foundNotes.forEach(note => {
timeoutAdder += 100
const newUpdate = new Promise((resolve, reject) => {
setTimeout(() => {
console.log('Encrypting Note ', note.id)
}) const created = Math.round((+new Date)/1000)
const salt = cs.createSmallSalt()
resolve('Its probably running... :-D') const noteText = note.text
}) const noteTitle = note.title
})
}
Note.fixAttachmentThumbnails = () => { const snippet = JSON.stringify([noteTitle, noteText.substring(0, 500)])
const filePath = '../staticFiles/' const noteSnippet = cs.encrypt(masterKey, salt, snippet)
db.promise()
.query(`SELECT * FROM attachment WHERE file_location NOT LIKE "%.%"`)
.then( (rows, fields) => {
rows[0].forEach(line => { const textObject = JSON.stringify([noteTitle, noteText])
const encryptedText = cs.encrypt(masterKey, salt, textObject)
const rawFilename = line['file_location'] db.promise()
const goodFileName = rawFilename+'.jpg' .query('UPDATE note_raw_text SET title = ?, text = ?, snippet = ?, salt = ? WHERE id = ?',
[null, encryptedText, noteSnippet, salt, note.note_raw_text_id])
//Rename file to have jpg extension, create thumbnail, update database .then(() => {
fs.rename(filePath+rawFilename, filePath+goodFileName, (err) => { resolve(true)
db.promise()
.query(`UPDATE attachment SET file_location = ? WHERE id = ?`,[goodFileName, line['id'] ])
.then( (rows, fields) => {
gm(filePath+goodFileName)
.resize(550) //Resize to width of 550 px
.quality(75) //compression level 0 - 100 (best)
.write(filePath + 'thumb_'+goodFileName, function (err) {
console.log('Done for -> ', goodFileName)
})
}) })
}, timeoutAdder)
}) })
allTheUpdates.push(newUpdate)
}) })
})
} Promise.all(allTheUpdates).then(done => {
Note.stressTest = () => { console.log('Indexing first 100')
return new Promise((resolve, reject) => { return Note.reindex(userId, masterKey)
db.promise()
.query(` }).then(results => {
SELECT text FROM note; console.log('Done')
resolve(true)
`)
.then((rows, fields) => {
console.log()
rows[0].forEach(item => {
Note.create(68, item['text'])
}) })
resolve(true)
}) })
.catch(console.log)
}) })
} }
// -------------- Note.create = (userId, noteTitle, noteText, masterKey) => {
Note.create = (userId, noteTitle, noteText, quickNote = 0, ) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if(userId == null || userId < 10){ reject('User Id required to create note') } if(userId == null || userId < 10){ reject('User Id required to create note') }
const created = Math.round((+new Date)/1000) const created = Math.round((+new Date)/1000)
const salt = cs.createSmallSalt()
const textObject = JSON.stringify([noteTitle, noteText])
const encryptedText = cs.encrypt(masterKey, salt, textObject)
db.promise() db.promise()
.query(`INSERT INTO note_raw_text (text, title, updated) VALUE (?, ?, ?)`, [noteText, noteTitle, created]) .query(`INSERT INTO note_raw_text (text, salt, updated) VALUE (?, ?, ?)`, [encryptedText, salt, created])
.then( (rows, fields) => { .then( (rows, fields) => {
const rawTextId = rows[0].insertId const rawTextId = rows[0].insertId
return db.promise() return db.promise()
.query('INSERT INTO note (user_id, note_raw_text_id, created, quick_note) VALUES (?,?,?,?)', .query('INSERT INTO note (user_id, note_raw_text_id, created, quick_note) VALUES (?,?,?,?)',
[userId, rawTextId, created, quickNote]) [userId, rawTextId, created, 0])
}) })
.then((rows, fields) => { .then((rows, fields) => {
// Indexing is done on save // Indexing is done on save
@@ -124,9 +114,193 @@ Note.create = (userId, noteTitle, noteText, quickNote = 0, ) => {
}) })
} }
Note.reindex = (userId, noteId) => { Note.reindex = (userId, masterKey) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if(!masterKey || masterKey.length == 0){
return reject('Master key needed for reindex')
}
let notIndexedNoteIds = []
let searchIndex = null
let searchIndexSalt = null
let foundNotes = null
//First check if we have any notes to index
db.promise().query(`
SELECT note.id, text, salt FROM note
JOIN note_raw_text ON note.note_raw_text_id = note_raw_text.id
WHERE indexed = 0 AND encrypted = 0 AND salt IS NOT NULL
AND user_id = ? LIMIT 100`, [userId])
.then((rows, fields) => {
//Halt execution if there are no new notes
foundNotes = rows[0]
if(foundNotes.length == 0){
throw new Error('No new notes to index')
}
//Select search index, if it doesn't exist, create it
return db.promise().query(`SELECT * FROM user_encrypted_search_index WHERE user_id = ? LIMIT 1`, [userId])
})
.then((rows, fields) => {
if(rows[0].length == 0){
console.log('Creating a new index')
//Create search index entry, return an object
searchIndexSalt = cs.createSmallSalt()
//Select all user notes to recreate index
return db.promise().query(`
SELECT note.id, text, salt FROM note
JOIN note_raw_text ON note.note_raw_text_id = note_raw_text.id
WHERE encrypted = 0 AND user_id = ?`, [userId])
.then((rows, fields) => {
foundNotes = rows[0]
return db.promise().query("INSERT INTO user_encrypted_search_index (`user_id`, `salt`) VALUES (?,?)", [userId, searchIndexSalt])
})
.then((rows, fields) => {
//return a fresh search index
return new Promise((resolve, reject) => { resolve('{}') })
})
} else {
const row = rows[0][0]
searchIndexSalt = row.salt
//Decrypt search index and continue.
let decipheredSearchIndex = '{}'
if(row.index && row.index.length > 0){
//Decrypt json, do not parse json yet, we want raw text
decipheredSearchIndex = cs.decrypt(masterKey, searchIndexSalt, row.index)
}
return new Promise((resolve, reject) => { resolve( decipheredSearchIndex ) })
}
})
.then(rawSearchIndex => {
searchIndex = rawSearchIndex
//Remove all instances of IDs from text
foundNotes.forEach(note => {
notIndexedNoteIds.push(note.id)
//Remove every instance of note id
const removeId = new RegExp(note.id,"gm")
const removeDoubles = new RegExp(',,',"g")
// const removeTrail = new RegExp(',]',"g")
searchIndex = searchIndex
.replace(removeId, '')
.replace(removeDoubles, ',')
.replace(/,]/g, ']')
.replace(/\[\,/g, '[') //search [,
})
searchIndex = JSON.parse(searchIndex)
//Remove unused words, this may not be needed and it increases overhead
Object.keys(searchIndex).forEach(word => {
if(searchIndex[word].length == 0){
delete searchIndex[word]
}
})
//Process text of each note and add it to the index
let reindexQueue = []
let reindexTimer = 0
foundNotes.forEach(note => {
reindexTimer += 50
let reindexPromise = new Promise((resolve, reject) => {
setTimeout(() => {
if(masterKey == null || note.salt == null){
console.log('Error indexing note', note.id)
return resolve(true)
}
const noteHtml = cs.decrypt(masterKey, note.salt, note.text)
const rawText =
ProcessText.removeHtml(noteHtml) //Remove HTML
.toLowerCase()
.replace(/style=".*?"/g,'') //Remove inline styles
.replace (/&#{0,1}[a-z0-9]+;/ig, '') //remove HTML entities
.replace(/[^A-Za-z0-9]/g, ' ') //Convert all to a-z only
.replace(/ +(?= )/g,'') //Remove double spaces
rawText.split(' ').forEach(word => {
//Skip small words
if(word.length <= 2){ return }
if(Array.isArray( searchIndex[word] )){
if(searchIndex[word].indexOf( note.id ) == -1){
searchIndex[word].push( note.id )
}
} else {
searchIndex[word] = [ note.id ]
}
})
return resolve(true)
}, reindexTimer)
})
reindexQueue.push(reindexPromise)
})
return Promise.all(reindexQueue)
})
.then(rawSearchIndex => {
console.log('All notes indexed')
const created = Math.round((+new Date)/1000)
const jsonSearchIndex = JSON.stringify(searchIndex)
const encryptedJsonIndex = cs.encrypt(masterKey, searchIndexSalt, jsonSearchIndex)
return db.promise().query("UPDATE user_encrypted_search_index SET `index` = ?, `last_update` = ? WHERE (`user_id` = ?) LIMIT 1",
[encryptedJsonIndex, created, userId])
.then((rows, fields) => {
return db.promise().query('UPDATE note SET `indexed` = 1 WHERE (`id` IN (?))', [notIndexedNoteIds])
})
.then((rows, fields) => {
console.log('Indexd Note Count: ' + rows[0]['affectedRows'])
resolve(true)
})
}).catch(error => {
console.log('Reindex Error')
console.log(error)
})
//Find all note Ids that need to be reindexed
// return resolve(true)
return
Note.get(userId, noteId) Note.get(userId, noteId)
.then(note => { .then(note => {
@@ -163,15 +337,9 @@ Note.reindex = (userId, noteId) => {
}) })
} }
Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, password = '', passwordHint = '') => { Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived, password = '', passwordHint = '', masterKey) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
//Prevent note loss if it saves with empty text
//if(ProcessText.removeHtml(noteText) == ''){
// console.log('Not saving empty note')
// resolve(false)
//}
const now = Math.round((+new Date)/1000) const now = Math.round((+new Date)/1000)
db.promise() db.promise()
@@ -183,11 +351,7 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived,
const textId = rows[0][0]['note_raw_text_id'] const textId = rows[0][0]['note_raw_text_id']
let salt = rows[0][0]['salt'] let salt = rows[0][0]['salt']
let noteSnippet = ''
//If password is removed, remove salt. generate a new one next time its encrypted
if(password.length == 0){
salt = null
}
//If a password is set, create a salt //If a password is set, create a salt
if(password.length > 3 && !salt){ if(password.length > 3 && !salt){
@@ -206,11 +370,20 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived,
// //
// @TODO - Do note save data if encryption goes wrong, do some validation // @TODO - Do note save data if encryption goes wrong, do some validation
// //
} else {
//Create encrypted snippet
const snippet = JSON.stringify([noteTitle, noteText.substring(0, 500)])
noteSnippet = cs.encrypt(masterKey, salt, snippet)
//Encrypt note text
const textObject = JSON.stringify([noteTitle, noteText])
noteText = cs.encrypt(masterKey, salt, textObject)
} }
//Update Note text //Update Note text
return db.promise() return db.promise()
.query('UPDATE note_raw_text SET text = ?, title = ?, updated = ?, salt = ? WHERE id = ?', [noteText, noteTitle, now, salt, textId]) .query('UPDATE note_raw_text SET text = ?, snippet = ? ,updated = ?, salt = ? WHERE id = ?', [noteText, noteSnippet, now, salt, textId])
}) })
.then( (rows, fields) => { .then( (rows, fields) => {
@@ -218,14 +391,14 @@ Note.update = (io, userId, noteId, noteText, noteTitle, color, pinned, archived,
//Update other note attributes //Update other note attributes
return db.promise() return db.promise()
.query('UPDATE note SET pinned = ?, archived = ?, color = ?, encrypted = ? WHERE id = ? AND user_id = ? LIMIT 1', .query('UPDATE note SET pinned = ?, archived = ?, color = ?, encrypted = ?, indexed = 0 WHERE id = ? AND user_id = ? LIMIT 1',
[pinned, archived, color, encrypted, noteId, userId]) [pinned, archived, color, encrypted, noteId, userId])
}) })
.then((rows, fields) => { .then((rows, fields) => {
//Async solr note reindex //Async solr note reindex
Note.reindex(userId, noteId) // Note.reindex(userId, noteId)
//Async attachment reindex //Async attachment reindex
Attachment.scanTextForWebsites(io, userId, noteId, noteText) Attachment.scanTextForWebsites(io, userId, noteId, noteText)
@@ -392,12 +565,16 @@ Note.getDiffText = (userId, noteId, usersCurrentText, lastUpdated) => {
} }
Note.get = (userId, noteId, password = '') => { Note.get = (userId, noteId, password = '', masterKey) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if(!masterKey || masterKey.length == 0){
return reject('Get note called without master key')
}
db.promise() db.promise()
.query(` .query(`
SELECT SELECT
note_raw_text.title,
note_raw_text.text, note_raw_text.text,
note_raw_text.salt, note_raw_text.salt,
note_raw_text.password_hint, note_raw_text.password_hint,
@@ -482,6 +659,17 @@ Note.get = (userId, noteId, password = '') => {
db.promise().query('UPDATE note_raw_text SET decrypt_attempts_count = decrypt_attempts_count +1 WHERE id = ?', [rawTextId ]) db.promise().query('UPDATE note_raw_text SET decrypt_attempts_count = decrypt_attempts_count +1 WHERE id = ?', [rawTextId ])
} }
} }
if(noteData.encrypted == 0 && noteData.salt && noteData.salt.length > 0){
//Normal Encrypted note
const decipheredText = cs.decrypt(masterKey, noteData.salt, noteData.text)
if(decipheredText == null){
throw new Error('Unable to decropt note text')
}
//Parse title and text from encrypted data and update object
const textObject = JSON.parse(decipheredText)
noteData.title = textObject[0]
noteData.text = textObject[1]
}
db.promise().query(`UPDATE note SET opened = ? WHERE (id = ?)`, [nowTime, noteId]) db.promise().query(`UPDATE note SET opened = ? WHERE (id = ?)`, [nowTime, noteId])
@@ -511,56 +699,75 @@ Note.getShared = (noteId) => {
} }
// Searches text index, returns nothing if there is no search query // Searches text index, returns nothing if there is no search query
Note.solrQuery = (userId, searchQuery, searchTags) => { Note.solrQuery = (userId, searchQuery, searchTags, masterKey) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if(searchQuery.length == 0){ if(searchQuery.length == 0){
resolve(null) resolve(null)
} else { } else {
//Number of characters before and after search word if(!masterKey || masterKey == null){
const front = 20 console.log('Attempting to search wiouth key')
const tail = 150 return resolve(null)
}
db.promise() //Search the search index
.query(` db.promise().query(`SELECT * FROM user_encrypted_search_index WHERE user_id = ? LIMIT 1`, [userId])
SELECT
note_id,
substring(
text,
IF(LOCATE(?, text) > ${tail}, LOCATE(?, text) - ${front}, 1),
${tail} + LENGTH(?) + ${front}
) as snippet
FROM note_text_index
WHERE user_id = ?
AND MATCH(text)
AGAINST(? IN NATURAL LANGUAGE MODE)
LIMIT 1000
;
`, [searchQuery, searchQuery, searchQuery, userId, searchQuery])
.then((rows, fields) => { .then((rows, fields) => {
let results = [] if(rows[0].length == 1){
let snippets = {}
rows[0].forEach(item => { //Lookup, decrypt and parse search index
let noteId = parseInt(item['note_id']) const row = rows[0][0]
//Setup array of ids to use for query const decipheredSearchIndex = cs.decrypt(masterKey, row.salt, row.index)
results.push( noteId ) const searchIndex = JSON.parse(decipheredSearchIndex)
//Get text snippet and highlight the key word
snippets[noteId] = item['snippet'].replace(new RegExp(searchQuery,"ig"), '<em>'+searchQuery+'</em>'); //Clean up search word
//.replace(searchQuery,'<em>'+searchQuery+'</em>') const word = searchQuery.toLowerCase().replace(/[^a-z0-9]/g, '')
})
let noteIds = []
let partials = []
Object.keys(searchIndex).forEach(wordIndex => {
if( wordIndex.indexOf(word) != -1 && wordIndex != word){
partials.push(wordIndex)
noteIds.push(...searchIndex[wordIndex])
}
})
const exactArray = searchIndex[word] ? searchIndex[word] : []
let searchData = {
'word':word,
'exact': exactArray,
'partials': partials,
'partial': [...new Set(noteIds) ],
}
//Remove exact matches from partials set if there is overlap
if(searchData['exact'].length > 0 && searchData['partial'].length > 0){
searchData['partial'] = searchData['partial']
.filter( ( el ) => !searchData['exact'].includes( el ) )
}
searchData['ids'] = searchData['exact'].concat(searchData['partial'])
searchData['total'] = searchData['ids'].length
console.log(searchData['total'])
return resolve({ 'ids':searchData['ids'] })
} else {
return resolve(null)
}
resolve({ 'ids':results, 'snippets':snippets })
}) })
} }
}) })
} }
Note.search = (userId, searchQuery, searchTags, fastFilters) => { Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
//Define return data objects //Define return data objects
let returnData = { let returnData = {
@@ -568,17 +775,16 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
'tags':[] 'tags':[]
} }
Note.solrQuery(userId, searchQuery, searchTags).then( (textSearchResults) => { Note.solrQuery(userId, searchQuery, searchTags, masterKey).then( (textSearchResults) => {
//Pull out search results from previous query //Pull out search results from previous query
let textSearchIds = [] let textSearchIds = []
let highlights = {}
let returnTagResults = false let returnTagResults = false
let searchAllNotes = false let searchAllNotes = false
if(textSearchResults != null){ if(textSearchResults != null){
textSearchIds = textSearchResults['ids'] textSearchIds = textSearchResults['ids']
highlights = textSearchResults['snippets'] // highlights = textSearchResults['snippets']
} }
//No results, return empty data //No results, return empty data
@@ -588,11 +794,14 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
// Base of the query, modified with fastFilters // Base of the query, modified with fastFilters
// Add to query for character counts -> CHAR_LENGTH(note.text) as chars // Add to query for character counts -> CHAR_LENGTH(note.text) as chars
//SUBSTRING(note_raw_text.text, 1, 500) as text,
let searchParams = [userId] let searchParams = [userId]
let noteSearchQuery = ` let noteSearchQuery = `
SELECT note.id, SELECT note.id,
SUBSTRING(note_raw_text.text, 1, 500) as text,
note_raw_text.title as title, note_raw_text.title as title,
note_raw_text.snippet as snippet,
note_raw_text.salt as salt,
note_raw_text.updated as updated, note_raw_text.updated as updated,
opened, opened,
color, color,
@@ -725,17 +934,24 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
//Grab note ID for finding tags //Grab note ID for finding tags
noteIds.push(note.id) noteIds.push(note.id)
if(note.text == null){ note.text = '' } if(note.encrypted == 1){
if(note.encrypted == 1){ note.text = '' } note.text = ''
}
//Decrypt note text
if(note.snippet && note.salt){
const decipheredText = cs.decrypt(masterKey, note.salt, note.snippet)
const textObject = JSON.parse(decipheredText)
if(textObject != null && textObject.length == 2){
note.title = textObject[0]
note.text = textObject[1]
}
}
//Deduce note title //Deduce note title
const textData = ProcessText.deduceNoteTitle(note.title, note.text) const textData = ProcessText.deduceNoteTitle(note.title, note.text)
// console.log(textData)
note.title = textData.title note.title = textData.title
note.subtext = textData.sub note.subtext = textData.sub
note.titleLength = textData.titleLength
note.subtextLength = textData.subtextLength
note.note_highlights = [] note.note_highlights = []
note.attachment_highlights = [] note.attachment_highlights = []
@@ -750,13 +966,10 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
note.thumbs = thumbArray note.thumbs = thumbArray
} }
//Push in search highlights
if(highlights && highlights[note.id]){
note['note_highlights'] = [highlights[note.id]]
}
//Clear out note.text before sending it to front end, its being used in title and subtext //Clear out note.text before sending it to front end, its being used in title and subtext
delete note.text delete note.text
delete note.snippet
delete note.salt
}) })
//If no notes are returned, there are no tags, return empty //If no notes are returned, there are no tags, return empty

132
server/models/User.js
View File

@@ -1,7 +1,10 @@
var crypto = require('crypto') var crypto = require('crypto')
let db = require('@config/database') const Note = require('@models/Note')
let Auth = require('@helpers/Auth')
const db = require('@config/database')
const Auth = require('@helpers/Auth')
const cs = require('@helpers/CryptoString')
let User = module.exports = {} let User = module.exports = {}
@@ -22,26 +25,32 @@ User.login = (username, password) => {
//User not found, create a new account with set data //User not found, create a new account with set data
if(rows[0].length == 0){ if(rows[0].length == 0){
User.create(lowerName, password) User.create(lowerName, password)
.then(loginToken => { .then( ({token, userId}) => {
resolve(loginToken) return resolve({ token, userId })
}) })
return
} }
//hash the password and check for a match if(lookedUpUser && lookedUpUser.salt){
const salt = new Buffer(lookedUpUser.salt, 'binary') //hash the password and check for a match
crypto.pbkdf2(password, salt, lookedUpUser.iterations, 512, 'sha512', function(err, delivered_key){ const salt = new Buffer(lookedUpUser.salt, 'binary')
if(delivered_key.toString('hex') === lookedUpUser.password){ crypto.pbkdf2(password, salt, lookedUpUser.iterations, 512, 'sha512', function(err, delivered_key){
if(delivered_key.toString('hex') === lookedUpUser.password){
//Passback a json web token User.generateMasterKey(lookedUpUser.id, password)
const token = Auth.createToken(lookedUpUser.id) .then( result => User.getMasterKey(lookedUpUser.id, password))
resolve(token) .then(masterKey => {
} else { //Passback a json web token
const token = Auth.createToken(lookedUpUser.id, masterKey)
resolve({ token: token, userId:lookedUpUser.id })
})
reject('Password does not match database') } else {
}
}) reject('Password does not match database')
}
})
}
}) })
.catch(console.log) .catch(console.log)
@@ -93,9 +102,15 @@ User.create = (username, password) => {
if(rows[0].affectedRows == 1){ if(rows[0].affectedRows == 1){
const newUserId = rows[0].insertId const userId = rows[0].insertId
const loginToken = Auth.createToken(newUserId)
resolve(loginToken) User.generateMasterKey(userId, password)
.then( result => User.getMasterKey(userId, password))
.then(masterKey => {
const token = Auth.createToken(userId, masterKey)
return resolve({token, userId})
})
} else { } else {
//Emit Error to user //Emit Error to user
@@ -168,3 +183,82 @@ User.getCounts = (userId) => {
}) })
} }
User.generateMasterKey = (userId, password) => {
return new Promise((resolve, reject) => {
if(!userId || !password){
reject('Need userId and password to generate key')
}
db.promise()
.query('SELECT count(id) as total FROM user_key WHERE user_id = ?', [userId])
.then((rows, fields) => {
//Entry already exists, you good.
if(rows[0][0]['total'] > 0){
return resolve(true)
// throw new Error('User Encryption key already exists')
} else {
// Generate user key, its big and random
const masterPassword = cs.createSmallSalt()
console.log('Generating new key for user', userId)
//Generate a salt because it wants it
const salt = cs.createSmallSalt()
// Encrypt master password
const encryptedMasterPassword = cs.encrypt(password, salt, masterPassword)
const created = Math.round((+new Date)/1000)
db.promise()
.query(
'INSERT INTO user_key (`user_id`, `salt`, `key`, `created`) VALUES (?, ?, ?, ?);',
[userId, salt, encryptedMasterPassword, created]
)
.then((rows, fields)=>{
return Note.encryptEveryNote(userId, masterPassword)
})
.then(results => {
return new Promise((resolve, reject) => { resolve(true) })
})
}
})
.then((rows, fields) => {
return resolve(true)
})
.catch(error => {
console.log('Create Master Password Error')
console.log(error)
})
})
}
User.getMasterKey = (userId, password) => {
if(!userId || !password){
reject('Need userId and password to fetch key')
}
return new Promise((resolve, reject) => {
db.promise().query('SELECT * FROM user_key WHERE user_id = ? LIMIT 1', [userId])
.then((rows, fields) => {
const row = rows[0][0]
const masterKey = cs.decrypt(password, row['salt'], row['key'])
if(masterKey == null){
return reject('Unable to decrypt key')
}
return resolve(masterKey)
})
})
}

26
server/routes/noteController.js
View File

@@ -5,15 +5,13 @@ let Notes = require('@models/Note');
let ShareNote = require('@models/ShareNote'); let ShareNote = require('@models/ShareNote');
let userId = null let userId = null
let socket = null let masterKey = null
// middleware that is specific to this router // middleware that is specific to this router
router.use(function setUserId (req, res, next) { router.use(function setUserId (req, res, next) {
if(req.headers.userId){ if(req.headers.userId){
userId = req.headers.userId userId = req.headers.userId
} masterKey = req.headers.masterKey
if(req.headers.socket){
// socket = req.
} }
next() next()
@@ -23,11 +21,8 @@ router.use(function setUserId (req, res, next) {
// Note actions // Note actions
// //
router.post('/get', function (req, res) { router.post('/get', function (req, res) {
// req.io.emit('welcome_homie', 'Welcome, dont poop from excitement') Notes.get(userId, req.body.noteId, req.body.password, masterKey)
Notes.get(userId, req.body.noteId, req.body.password)
.then( data => { .then( data => {
//Join room when user opens note
// req.io.join('note_room')
res.send(data) res.send(data)
}) })
}) })
@@ -38,17 +33,17 @@ router.post('/delete', function (req, res) {
}) })
router.post('/create', function (req, res) { router.post('/create', function (req, res) {
Notes.create(userId, req.body.title, req.body.text) Notes.create(userId, req.body.title, req.body.text, masterKey)
.then( id => res.send({id}) ) .then( id => res.send({id}) )
}) })
router.post('/update', function (req, res) { router.post('/update', function (req, res) {
Notes.update(req.io, userId, req.body.noteId, req.body.text, req.body.title, req.body.color, req.body.pinned, req.body.archived, req.body.password, req.body.hint) Notes.update(req.io, userId, req.body.noteId, req.body.text, req.body.title, req.body.color, req.body.pinned, req.body.archived, req.body.password, req.body.hint, masterKey)
.then( id => res.send({id}) ) .then( id => res.send({id}) )
}) })
router.post('/search', function (req, res) { router.post('/search', function (req, res) {
Notes.search(userId, req.body.searchQuery, req.body.searchTags, req.body.fastFilters) Notes.search(userId, req.body.searchQuery, req.body.searchTags, req.body.fastFilters, masterKey)
.then( notesAndTags => { .then( notesAndTags => {
res.send(notesAndTags) res.send(notesAndTags)
}) })
@@ -62,6 +57,14 @@ router.post('/difftext', function (req, res) {
}) })
}) })
router.post('/reindex', function (req, res) {
Notes.reindex(userId, masterKey)
.then( data => {
res.send(data)
})
})
// //
// Update single note attributes // Update single note attributes
// //
@@ -116,5 +119,4 @@ router.get('/reindex5yu43prchuj903mrc', function (req, res) {
}) })
module.exports = router module.exports = router

23
server/routes/userController.js
View File

@@ -2,6 +2,7 @@ var express = require('express')
var router = express.Router() var router = express.Router()
let User = require('@models/User'); let User = require('@models/User');
const cs = require('@helpers/CryptoString')
// middleware that is specific to this router // middleware that is specific to this router
router.use(function timeLog (req, res, next) { router.use(function timeLog (req, res, next) {
@@ -31,19 +32,19 @@ router.post('/login', function (req, res) {
} }
User.login(username, password) User.login(username, password)
.then(function(loginToken){ .then( ({token, userId}) => {
//Return json web token to user returnData['username'] = username
returnData['success'] = true returnData['token'] = token
returnData['token'] = loginToken returnData['success'] = true
returnData['username'] = username
res.send(returnData) res.send(returnData)
}) return
.catch(e => { })
console.log(e) .catch(e => {
res.send(returnData) console.log(e)
}) res.send(returnData)
})
}) })
// fetch counts of users notes // fetch counts of users notes