let db = require('@config/database') let Tags = require('@models/Tag') let Attachment = require('@models/Attachment') let ProcessText = require('@helpers/ProcessText') const DiffMatchPatch = require('@helpers/DiffMatchPatch') var rp = require('request-promise'); const fs = require('fs') let Note = module.exports = {} Note.stressTest = () => { return new Promise((resolve, reject) => { db.promise() .query(` SELECT text FROM note; `) .then((rows, fields) => { console.log() rows[0].forEach(item => { Note.create(68, item['text']) }) resolve(true) }) .catch(console.log) }) } Note.create = (userId, noteText, quickNote = 0) => { return new Promise((resolve, reject) => { if(userId == null || userId < 10){ reject('User Id required to create note') } const created = Math.round((+new Date)/1000) db.promise() .query('INSERT INTO note (user_id, text, updated, created, quick_note) VALUES (?,?,?,?,?)', [userId, noteText, created, created, quickNote]) .then((rows, fields) => { // New notes are empty, don't add to solr index // Note.reindex(userId, rows[0].insertId) resolve(rows[0].insertId) //Only return the new note ID when creating a new note }) .catch(console.log) }) } Note.reindexAll = () => { return new Promise((resolve, reject) => { db.promise() .query(` SELECT id, user_id FROM note; `) .then((rows, fields) => { console.log() rows[0].forEach(item => { Note.reindex(item.user_id, item.id) }) resolve(true) }) .catch(console.log) }) } Note.reindex = (userId, noteId) => { return new Promise((resolve, reject) => { Note.get(userId, noteId) .then(note => { const noteText = note.text //Process note text and attachment data Attachment.scanTextForWebsites(userId, noteId, noteText) .then( allNoteAttachmentText => { // // Update Solr index // Tags.string(userId, noteId) .then(tagString => { const fullText = ProcessText.removeHtml(noteText) +' '+ tagString +' '+ ProcessText.removeHtml(allNoteAttachmentText) db.promise() .query(` INSERT INTO note_text_index (note_id, user_id, text) VALUES (?,?,?) ON DUPLICATE KEY UPDATE text = ? `, [noteId, userId, fullText, fullText]) .then((rows, fields) => { resolve(true) }) .catch(console.log) }) }) }) }) } Note.update = (userId, noteId, noteText, color, pinned, archived) => { return new Promise((resolve, reject) => { //Prevent note loss if it saves with empty text if(ProcessText.removeHtml(noteText) == ''){ console.log('Not saving empty note') resolve(false) } const now = Math.round((+new Date)/1000) db.promise() .query('UPDATE note SET text = ?, pinned = ?, archived = ?, updated = ?, color = ? WHERE id = ? AND user_id = ? LIMIT 1', [noteText, pinned, archived, now, color, noteId, userId]) .then((rows, fields) => { //Async solr note reindex Note.reindex(userId, noteId) //Send back updated response resolve(rows[0]) }) .catch(console.log) }) } // // Delete a note and all its remaining parts // Note.delete = (userId, noteId) => { return new Promise((resolve, reject) => { db.promise().query('DELETE FROM note WHERE note.id = ? AND note.user_id = ?', [noteId,userId]) .then((rows, fields) => { return db.promise().query('DELETE FROM note_text_index WHERE note_text_index.note_id = ? AND note_text_index.user_id = ?', [noteId,userId]) }) .then((rows, fields) => { //Select all attachments with files return db.promise().query('SELECT file_location FROM attachment WHERE attachment.note_id = ? AND attachment.user_id = ?', [noteId,userId]) }) .then((attachmentRows, fields) => { //Go through each selected attachment and delete the files attachmentRows[0].forEach( location => { const fileName = location['file_location'] if(fileName != null && fileName.length > 1){ fs.unlink('../staticFiles/'+fileName ,function(err){ //Async, just rip through them. if(err) return console.log(err); // console.log('file deleted successfully => ', fileName); }) } }) return db.promise().query('DELETE FROM attachment WHERE attachment.note_id = ? AND attachment.user_id = ?', [noteId,userId]) }) .then((rows, fields) => { return db.promise().query('DELETE FROM note_tag WHERE note_tag.note_id = ? AND note_tag.user_id = ?', [noteId,userId]) }) .then((rows, fields) => { resolve(true) }) }) } //text is the current text for the note that will be compared to the text in the database Note.getDiffText = (userId, noteId, usersCurrentText, lastUpdated) => { return new Promise((resolve, reject) => { Note.get(userId, noteId) .then(noteObject => { let oldText = noteObject.text.replace(/(\r\n|\n|\r)/gm,"") let newText = usersCurrentText.replace(/(\r\n|\n|\r)/gm,"") if(noteObject.updated == lastUpdated){ // console.log('No note diff') resolve(null) } if(noteObject.updated > lastUpdated){ newText = noteObject.text.replace(/(\r\n|\n|\r)/gm,"") oldText = usersCurrentText.replace(/(\r\n|\n|\r)/gm,"") } const dmp = new DiffMatchPatch.diff_match_patch() const diff = dmp.diff_main(oldText, newText) dmp.diff_cleanupSemantic(diff) const patch_list = dmp.patch_make(oldText, newText, diff); const patch_text = dmp.patch_toText(patch_list); //Patch text - shows a list of changes var patches = dmp.patch_fromText(patch_text); // console.log(patch_text) //results[1] - contains diagnostic data for patch apply, its possible it can fail var results = dmp.patch_apply(patches, oldText); //Compile return data for front end const returnData = { updatedText: results[0], diffs: results[1].length, //Only use length for now updated: Math.max(noteObject.updated,lastUpdated) //Return most recently updated date } //Final change in notes // console.log(returnData) resolve(returnData) }) }) } Note.get = (userId, noteId) => { return new Promise((resolve, reject) => { db.promise() .query(` SELECT note.text, note.updated, note.pinned, note.archived, note.color, count(distinct attachment.id) as attachment_count FROM note LEFT JOIN attachment ON (note.id = attachment.note_id) WHERE note.user_id = ? AND note.id = ? LIMIT 1`, [userId,noteId]) .then((rows, fields) => { //Return note data resolve(rows[0][0]) }) .catch(console.log) }) } Note.getShared = (noteId) => { return new Promise((resolve, reject) => { db.promise() .query('SELECT text, updated, color FROM note WHERE id = ? AND shared = 1 LIMIT 1', [noteId]) .then((rows, fields) => { //Return note data resolve(rows[0][0]) }) .catch(console.log) }) } // Searches text index, returns nothing if there is no search query Note.solrQuery = (userId, searchQuery, searchTags) => { return new Promise((resolve, reject) => { if(searchQuery.length == 0){ resolve(null) } else { //Number of characters before and after search word const front = 5 const tail = 150 db.promise() .query(` SELECT note_id, substring( text, IF(LOCATE(?, text) > ${tail}, LOCATE(?, text) - ${front}, 1), ${tail} + LENGTH(?) + ${front} ) as snippet FROM note_text_index WHERE user_id = ? AND MATCH(text) AGAINST(? IN NATURAL LANGUAGE MODE) LIMIT 1000 ; `, [searchQuery, searchQuery, searchQuery, userId, searchQuery]) .then((rows, fields) => { let results = [] let snippets = {} rows[0].forEach(item => { let noteId = parseInt(item['note_id']) //Setup array of ids to use for query results.push( noteId ) //Get text snippet and highlight the key word snippets[noteId] = item['snippet'].replace(new RegExp(searchQuery,"ig"), ''+searchQuery+''); //.replace(searchQuery,''+searchQuery+'') }) resolve({ 'ids':results, 'snippets':snippets }) }) } }) } Note.search = (userId, searchQuery, searchTags, fastFilters) => { return new Promise((resolve, reject) => { //Define return data objects let returnData = { 'notes':[], 'tags':[] } Note.solrQuery(userId, searchQuery, searchTags).then( (textSearchResults) => { //Pull out search results from previous query let textSearchIds = [] let highlights = {} let returnTagResults = false if(textSearchResults != null){ textSearchIds = textSearchResults['ids'] highlights = textSearchResults['snippets'] } //No results, return empty data if(textSearchIds.length == 0 && searchQuery.length > 0){ return resolve(returnData) } // Base of the query, modified with fastFilters // Add to query for character counts -> CHAR_LENGTH(note.text) as chars let noteSearchQuery = ` SELECT note.id, SUBSTRING(note.text, 1, 1500) as text, updated, color, count(distinct note_tag.id) as tag_count, count(distinct attachment.id) as attachment_count, note.pinned, note.archived FROM note LEFT JOIN note_tag ON (note.id = note_tag.note_id) LEFT JOIN attachment ON (note.id = attachment.note_id) WHERE note.user_id = ?` let searchParams = [userId] //If text search returned results, limit search to those ids if(textSearchIds.length > 0){ searchParams.push(textSearchIds) noteSearchQuery += ' AND note.id IN (?)' } if(fastFilters.noteIdSet && fastFilters.noteIdSet.length > 0){ searchParams.push(fastFilters.noteIdSet) noteSearchQuery += ' AND note.id IN (?)' } //If tags are passed, use those tags in search if(searchTags.length > 0){ searchParams.push(searchTags) noteSearchQuery += ' AND note_tag.tag_id IN (?)' } //Show archived notes, only if fast filter is set, default to not archived if(fastFilters.onlyArchived == 1){ noteSearchQuery += ' AND note.archived = 1' //Show Archived } else { noteSearchQuery += ' AND note.archived = 0' //Exclude archived } //Finish up note query noteSearchQuery += ' GROUP BY note.id' //Only show notes with Tags if(fastFilters.withTags == 1){ returnTagResults = true noteSearchQuery += ' HAVING tag_count > 0' } //Only show notes with links if(fastFilters.withLinks == 1){ returnTagResults = true noteSearchQuery += ' HAVING attachment_count > 0' } //Only show archived notes if(fastFilters.onlyArchived == 1){ returnTagResults = true noteSearchQuery += ' HAVING note.archived = 1' } // // Always prioritize pinned notes in searches. //Default Sort, order by last updated let defaultOrderBy = ' ORDER BY note.pinned DESC, updated DESC, created DESC, opened DESC, id DESC' //Order by Last Created Date if(fastFilters.lastCreated == 1){ defaultOrderBy = ' ORDER BY note.pinned DESC, created DESC, updated DESC, opened DESC, id DESC' } //Order by last Opened Date if(fastFilters.lastOpened == 1){ defaultOrderBy = ' ORDER BY note.pinned DESC, opened DESC, updated DESC, created DESC, id DESC' } //Append Order by to query noteSearchQuery += defaultOrderBy //Manage limit params if set if(fastFilters.limitSize > 0 || fastFilters.limitOffset > 0){ const limitSize = parseInt(fastFilters.limitSize, 10) || 10 //Use int or default to 10 const limitOffset = parseInt(fastFilters.limitOffset, 10) || 0 //Either parse int, or use zero // console.log(` LIMIT ${limitOffset}, ${limitSize}`) noteSearchQuery += ` LIMIT ${limitOffset}, ${limitSize}` } // console.log('------------- Final Query --------------') // console.log(noteSearchQuery) // console.log('------------- ----------- --------------') db.promise() .query(noteSearchQuery, searchParams) .then((noteRows, noteFields) => { //Push all notes returnData['notes'] = noteRows[0] //pull out all note ids so we can fetch all tags for those notes let noteIds = [] returnData['notes'].forEach(note => { //Grab note ID for finding tags noteIds.push(note.id) if(note.text == null){ note.text = '' } //Deduce note title const textData = ProcessText.deduceNoteTitle(note.text) // console.log(textData) note.title = textData.title note.subtext = textData.sub note.titleLength = textData.titleLength note.subtextLength = textData.subtextLength note.note_highlights = [] note.attachment_highlights = [] note.tag_highlights = [] //Push in search highlights if(highlights && highlights[note.id]){ note['note_highlights'] = [highlights[note.id]] } //Clear out note.text before sending it to front end, its being used in title and subtext delete note.text }) //If no notes are returned, there are no tags, return empty if(noteIds.length == 0){ return resolve(returnData) } //Return all notes, tags are not being searched // if tags are being searched, continue // if notes are being filtered, return tags if(searchTags.length == 0 && returnTagResults == false){ return resolve(returnData) } //Only show tags of selected notes db.promise() .query(`SELECT tag.id, tag.text, count(tag.id) as usages FROM note_tag JOIN tag ON (tag.id = note_tag.tag_id) WHERE note_tag.user_id = ? AND note_id IN (?) GROUP BY tag.id ORDER BY usages DESC;`,[userId, noteIds]) .then((tagRows, tagFields) => { returnData['tags'] = tagRows[0] resolve(returnData) }) .catch(console.log) }) .catch(console.log) }) .catch(console.log) }) }