I swear, I'm going to start doing regular commits

+ Added a ton of shit
+ About to add socket.io oh god.

server/helpers/ProcessText.js

@@ -2,8 +2,140 @@
 let ProcessText = module.exports = {}
 
 ProcessText.removeHtml = (string) => {
+
+	if(string == undefined || string == null || string.length == 0){
+		return ''
+	}
+
 	return string
 		.replace(/&[#A-Za-z0-9]+;/g,' ') //Rip out all HTML entities
 		.replace(/<[^>]+>/g, ' ') //Rip out all HTML tags
 		.replace(/\s+/g, ' ') //Remove all whitespace
+		.trim()
+}
+
+ProcessText.getUrlsFromString = (string) => {
+	const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
+	return string.match(urlPattern)
+}
+
+/* 
+	Pulls out title and subtext of note 
+		+ Title is always first line
+		+ Empty lines are skipped
+		+ URLs are turned into links
+		+ All URLs are givent the target="_blank" property
+*/
+
+ProcessText.deduceNoteTitle = (inString) => {
+
+	let title = '' //Title of note
+	let sub = '' //sub text below note
+
+	if(!inString || inString == null || inString.length == 0){
+		return {title, sub}
+	}
+
+	let lines = inString.match(/[^\r\n]+/g)
+	let finalLines = []
+
+	const startTags = ['<ol','<li','<ul']
+	const endTags = ['</o','</l','</u']
+
+	let totalLines = Math.min(lines.length, 6)
+	let charLimit = 250
+	let listStart = false
+
+	for(let i=0; i < totalLines; i++){
+
+		//Just in case 'i' gets bigger than array
+		if(lines[i] === undefined){
+			continue
+		}
+
+		const cleanLine = ProcessText.removeHtml(lines[i]).trim().replace('&nbsp','')
+		const lineStart = lines[i].trim().substring(0, 3)
+		charLimit -= cleanLine.length
+
+		//Close out list if char limit is hit
+		if(charLimit <= 0 && listStart){
+			finalLines.push(lines[i])
+			break
+		}
+
+		//Images appear as empty, push em!
+		if(cleanLine.length == 0 && lines[i].indexOf('<img') != -1){
+			finalLines.push(lines[i])
+			continue
+		}
+
+		//Empty line, may be a list open or close
+		if(cleanLine.length == 0 && (startTags.includes(lineStart) || endTags.includes(lineStart) )){
+			if(listStart == false){
+				charLimit = 400 //Double size for list notes
+			}
+			finalLines.push(lines[i])
+			totalLines++
+			listStart = true
+			continue
+		}
+
+		//If line is part of a list, up counter, we want the whole list
+		if(startTags.includes(lineStart)){
+			totalLines++
+		}
+
+		//Skip empty lines
+		if(!cleanLine || cleanLine.length == 0 || cleanLine == '&nbsp;'){
+			totalLines++
+			continue
+		}
+
+		//turn urls into links, don't process if its already an <a href=
+		const containsUrls = ProcessText.getUrlsFromString(cleanLine)
+		if(containsUrls && containsUrls.length == 1 && lines[i].indexOf('</a>') == -1){
+			const url = containsUrls[0]
+			lines[i] = lines[i].replace(url, `<a href="${url}">${url}</a>`)
+		}
+
+		//Insert target=_blank into links if set, do it for every link in line
+		if(lines[i].indexOf('</a>') > 0){
+			lines[i] = lines[i].replace(/<a /g, '<a target="_blank" ')
+		}
+
+		//Limit output characters
+		//Check character limit
+		if(charLimit <= 0 && listStart == false){
+
+			//Cut the string down to character limit
+			const cutString = lines[i].substring(0, lines[i].length+charLimit)
+			//Find last space and cut off everything after it
+			let cleanCutString = cutString.substring(0, cutString.lastIndexOf(' '))
+
+			//Some strings may not contain a space resulting in no string
+			if(cleanCutString.length == 0){
+				cleanCutString = cutString
+			}
+
+			finalLines.push(cleanCutString + '...')
+			break;
+		}
+
+		finalLines.push(lines[i])
+
+	}
+
+	//Pull out title if its not an empty string
+	if(ProcessText.removeHtml(finalLines[0]).trim().replace('&nbsp','').length > 0){
+		title = finalLines.shift()
+	}
+	
+	sub = finalLines.join('')
+
+	//Return final display lengths
+	let titleLength = ProcessText.removeHtml(title).trim().replace('&nbsp','').length
+	let subtextLength = ProcessText.removeHtml(sub).trim().replace('&nbsp','').length
+
+
+	return { title, sub, titleLength, subtextLength }
 }

server/models/Attachment.js

@@ -117,6 +117,7 @@ Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
 		Attachment.urlForNote(userId, noteId).then(attachments => {
 
 			//Find all URLs in text
+			//@TODO - Use the process text library for this function
 			const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
 			let allUrls = noteText.match(urlPattern)
 

server/models/Note.js

@@ -5,7 +5,10 @@ let Attachment = require('@models/Attachment')
 
 let ProcessText = require('@helpers/ProcessText')
 
+const DiffMatchPatch = require('@helpers/DiffMatchPatch')
+
 var rp = require('request-promise');
+const fs = require('fs')
 
 let Note = module.exports = {}
 
@@ -137,21 +140,96 @@ Note.update = (userId, noteId, noteText, color, pinned, archived) => {
 	})
 }
 
+
+//
+// Delete a note and all its remaining parts
+//
 Note.delete = (userId, noteId) => {
 	return new Promise((resolve, reject) => {
-	db.promise().query('DELETE FROM note WHERE note.id = ? AND note.user_id = ?', [noteId,userId])
-	.then((rows, fields) => {
-		db.promise().query('DELETE FROM attachment WHERE attachment.note_id = ? AND attachment.user_id = ?', [noteId,userId])
-		.then((rows, fields)=> {
-			db.promise().query('DELETE FROM note_tag WHERE note_tag.note_id = ? AND note_tag.user_id = ?', [noteId,userId])
-				.then((rows, fields)=> {
-					resolve(true)
-				})
+
+		db.promise().query('DELETE FROM note WHERE note.id = ? AND note.user_id = ?', [noteId,userId])
+		.then((rows, fields) => {
+			return db.promise().query('DELETE FROM note_text_index WHERE note_text_index.note_id = ? AND note_text_index.user_id = ?', [noteId,userId])	
+		})
+		.then((rows, fields) => {
+			//Select all attachments with files
+			return db.promise().query('SELECT file_location FROM attachment WHERE attachment.note_id = ? AND attachment.user_id = ?', [noteId,userId])
+		})
+		.then((attachmentRows, fields) => {
+
+			//Go through each selected attachment and delete the files
+			attachmentRows[0].forEach( location => {
+				const fileName = location['file_location']
+				if(fileName != null && fileName.length > 1){
+					fs.unlink('../staticFiles/'+fileName ,function(err){ //Async, just rip through them.
+						if(err) return console.log(err);
+						// console.log('file deleted successfully => ', fileName);
+					})
+				}	
 			})
+
+			return db.promise().query('DELETE FROM attachment WHERE attachment.note_id = ? AND attachment.user_id = ?', [noteId,userId])
+		})
+		.then((rows, fields) => {
+			return db.promise().query('DELETE FROM note_tag WHERE note_tag.note_id = ? AND note_tag.user_id = ?', [noteId,userId])	
+		})
+		.then((rows, fields) => {
+			resolve(true)
 		})
 	})
 }
 
+//text is the current text for the note that will be compared to the text in the database
+Note.getDiffText = (userId, noteId, usersCurrentText, lastUpdated) => {
+	return new Promise((resolve, reject) => {
+		Note.get(userId, noteId)
+		.then(noteObject => {
+
+
+			let oldText = noteObject.text.replace(/(\r\n|\n|\r)/gm,"")
+			let newText = usersCurrentText.replace(/(\r\n|\n|\r)/gm,"")
+
+			if(noteObject.updated == lastUpdated){
+				console.log('No note diff')
+				resolve(null)
+			}
+
+			if(noteObject.updated > lastUpdated){
+				newText = noteObject.text.replace(/(\r\n|\n|\r)/gm,"")
+				oldText = usersCurrentText.replace(/(\r\n|\n|\r)/gm,"")
+			}
+
+			const dmp = new DiffMatchPatch.diff_match_patch()
+			const diff = dmp.diff_main(oldText, newText)
+
+			dmp.diff_cleanupSemantic(diff)
+			const patch_list = dmp.patch_make(oldText, newText, diff);
+  			const patch_text = dmp.patch_toText(patch_list);
+  			
+  			//Patch text -  shows a list of changes
+  			var patches = dmp.patch_fromText(patch_text);
+  			// console.log(patch_text)
+
+  			//results[1] - contains diagnostic data for patch apply, its possible it can fail
+  			var results = dmp.patch_apply(patches, oldText);
+  			
+  			//Compile return data for front end
+  			const returnData = {
+  				updatedText: results[0],
+  				diffs: results[1].length, //Only use length for now
+  				updated: Math.max(noteObject.updated,lastUpdated) //Return most recently updated date
+
+  			}
+  			
+  			//Final change in notes
+  			console.log(returnData)
+
+			resolve(returnData)
+		})
+	})
+
+}
+
 Note.get = (userId, noteId) => {
 	return new Promise((resolve, reject) => {
 		db.promise()
@@ -184,6 +262,7 @@ Note.getShared = (noteId) => {
 	})
 }
 
+// Searches text index, returns nothing if there is no search query
 Note.solrQuery = (userId, searchQuery, searchTags) => {
 	return new Promise((resolve, reject) => {
 
@@ -243,27 +322,28 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
 
 		Note.solrQuery(userId, searchQuery, searchTags).then( (textSearchResults) => {
 
+			//Pull out search results from previous query
 			let textSearchIds = []
 			let highlights = {}
+			let returnTagResults = false
 
 			if(textSearchResults != null){
 				textSearchIds = textSearchResults['ids']
 				highlights = textSearchResults['snippets']
 			}
 
-			
-
 		    //No results, return empty data
 			if(textSearchIds.length == 0 && searchQuery.length > 0){
 				return resolve(returnData)
 			}
 
-			//Default note lookup gets all notes
+			// Base of the query, modified with fastFilters
 			// Add to query for character counts -> CHAR_LENGTH(note.text) as chars
 			let noteSearchQuery = `
 				SELECT note.id, 
-					SUBSTRING(note.text, 1, 400) as text, 
-					updated, color, 
+					SUBSTRING(note.text, 1, 1500) as text, 
+					updated, 
+					color, 
 					count(distinct note_tag.id) as tag_count, 
 					count(distinct attachment.id) as attachment_count,
 					note.pinned,
@@ -274,22 +354,26 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
 				WHERE note.user_id = ?`
 			let searchParams = [userId]
 
+			//If text search returned results, limit search to those ids			
 			if(textSearchIds.length > 0){
 				searchParams.push(textSearchIds)
 				noteSearchQuery += ' AND note.id IN (?)'
 			}
 
+			if(fastFilters.noteIdSet && fastFilters.noteIdSet.length > 0){
+				searchParams.push(fastFilters.noteIdSet)
+				noteSearchQuery += ' AND note.id IN (?)'
+			}
+
+			//If tags are passed, use those tags in search
 			if(searchTags.length > 0){
-				//If tags are passed, use those tags in search
 				searchParams.push(searchTags)
 				noteSearchQuery += ' AND note_tag.tag_id IN (?)'
 			}
 
-			//Toggle archived, show archived if tags are searched
-			// - archived will show archived in search results
-			// - onlyArchive will exclude notes that are not archived
-			if(fastFilters.archived == 1 || searchTags.length > 0 || fastFilters.onlyArchived == 1){
-				//Do nothing
+			//Show archived notes, only if fast filter is set, default to not archived
+			if(fastFilters.onlyArchived == 1){
+				noteSearchQuery += ' AND note.archived = 1' //Show Archived
 			} else {
 				noteSearchQuery += ' AND note.archived = 0' //Exclude archived
 			}
@@ -299,14 +383,17 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
 
 			//Only show notes with Tags
 			if(fastFilters.withTags == 1){
+				returnTagResults = true
 				noteSearchQuery += ' HAVING tag_count > 0'
 			}
 			//Only show notes with links
 			if(fastFilters.withLinks == 1){
+				returnTagResults = true
 				noteSearchQuery += ' HAVING attachment_count > 0'
 			}
 			//Only show archived notes
 			if(fastFilters.onlyArchived == 1){
+				returnTagResults = true
 				noteSearchQuery += ' HAVING note.archived = 1'
 			}
 
@@ -336,10 +423,13 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
 				const limitOffset = parseInt(fastFilters.limitOffset, 10) || 0 //Either parse int, or use zero
 
 
-				console.log(` LIMIT ${limitOffset}, ${limitSize}`)
+				// console.log(` LIMIT ${limitOffset}, ${limitSize}`)
 				noteSearchQuery += ` LIMIT ${limitOffset}, ${limitSize}`
 			}
 			
+			// console.log('------------- Final Query --------------')
+			// console.log(noteSearchQuery)
+			// console.log('------------- ----------- --------------')
 
 			db.promise()
 			.query(noteSearchQuery, searchParams)
@@ -356,38 +446,27 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
 					noteIds.push(note.id)
 
 					if(note.text == null){ note.text = '' }
+
+					//Deduce note title
+					const textData = ProcessText.deduceNoteTitle(note.text)
+
+					// console.log(textData)
 					
-					//Attempt to pull string out of first tag in note
-					let reg = note.text.match(/<([\w]+)[^>]*>(.*?)<\/\1>/g)
-
-					//Pull out first html tag contents, that is the title
-					if(reg != null && reg[0]){
-						note.title = reg[0] //First line from HTML
-					} else {
-						note.title = note.text //Entire note
-					}
-
-					//Clean up html title 
-					note.title = ProcessText.removeHtml(note.title)
-
-					//Generate Subtext
-					note.subtext = ''
-					if(note.text != '' && note.title != ''){
-						note.subtext = ProcessText.removeHtml(note.text)
-							.substring(note.title.length)
-					}
-					
+					note.title = textData.title
+					note.subtext = textData.sub
+					note.titleLength = textData.titleLength
+					note.subtextLength = textData.subtextLength
 
 					note.note_highlights = []
 					note.attachment_highlights = []
 					note.tag_highlights = []
 
-					//Push in solr highlights
+					//Push in search highlights
 					if(highlights && highlights[note.id]){
 						note['note_highlights'] = [highlights[note.id]]
 					}
 
-					//Clear out note.text before sending it to front end
+					//Clear out note.text before sending it to front end, its being used in title and subtext
 					delete note.text
 				})
 
@@ -396,6 +475,13 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
 					return resolve(returnData)
 				}
 
+				//Return all notes, tags are not being searched
+				// if tags are being searched, continue
+				// if notes are being filtered, return tags
+				if(searchTags.length == 0 && returnTagResults == false){
+					return resolve(returnData)
+				}
+
 				//Only show tags of selected notes
 				db.promise()
 				.query(`SELECT tag.id, tag.text, count(tag.id) as usages FROM note_tag

server/models/Tag.js

@@ -2,6 +2,21 @@ let db = require('@config/database')
 
 let Tag = module.exports = {}
 
+Tag.userTags = (userId) => {
+	return new Promise((resolve, reject) => {
+		db.promise()
+		.query(`
+			SELECT tag.id, text, COUNT(note_tag.note_id) as usages FROM tag
+			JOIN note_tag ON tag.id = note_tag.tag_id
+			WHERE note_tag.user_id = ?
+			GROUP BY tag.id
+			ORDER BY usages DESC
+		`, [userId])
+		.then( (rows, fields) => {
+			resolve(rows[0])
+		})
+	})
+}
 
 Tag.removeTagFromNote = (userId, tagId) => {
 	return new Promise((resolve, reject) => {

server/routes/attachmentController.js

@@ -1,7 +1,7 @@
 let express = require('express')
 
 var multer  = require('multer')
-var upload = multer({ dest: '../staticFiles/' })
+var upload = multer({ dest: '../staticFiles/' }) //@TODO make this a global value
 let router = express.Router()
 
 let Attachment = require('@models/Attachment');

server/routes/noteController.js

@@ -38,6 +38,15 @@ router.post('/search', function (req, res) {
 	.then( notesAndTags => res.send(notesAndTags))
 })
 
+router.post('/difftext', function (req, res) {
+
+	Notes.getDiffText(userId, req.body.noteId, req.body.text, req.body.updated)
+	.then( fullDiffText => {
+		//Response should be full diff text
+		res.send(fullDiffText)
+	})
+})
+
 //Reindex all notes. Not a very good function, not public
 router.get('/reindex5yu43prchuj903mrc', function (req, res) {
 

server/routes/tagController.js

@@ -42,4 +42,10 @@ router.post('/get', function (req, res) {
 	.then( data => res.send(data) )
 })
 
+//Get all the tags for this user in order of usage
+router.post('/usertags', function (req, res) {
+	Tags.userTags(userId)
+	.then( data => res.send(data) )
+})
+
 module.exports = router