Max/SolidScribe
1
0
Fork 0
Code Issues 24 Pull Requests Releases Wiki Activity

Searching, url indexing

Browse Source 
* Added a help page
* Cleaned up home and login pages
* Menu is hidden when on notes section of app
* Added username to login data
* Notes now change to the color selected for the note
* Note save function has a 500ms debounce to prevent spamming
* Solr results now displays content from notes, tags and attachments
* All note data is now indexed in solr
* Notes containing URLs are now scraped and put into tag solr index
* Attachments that are removed from note are deleted when url is removed
* Minor little tweaks and fixes all over the place
This commit is contained in:
Max G
2019-07-24 18:06:50 +00:00
parent e52ae65a42
commit b0a8071b41
16 changed files with 571 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
client/src/App.vue
View File

@@ -1,14 +1,25 @@
<template> <template>
<div id="app"> <div id="app">
<link href="https://fonts.googleapis.com/css?family=Open+Sans&display=swap" rel="stylesheet">
<div>
<router-link class="ui button" to="/">Home</router-link>
<router-link v-if="loggedIn" class="ui button" to="/notes">Notes</router-link> <link href="https://fonts.googleapis.com/css?family=Open+Sans&display=swap" rel="stylesheet">
<router-link v-if="!loggedIn" class="ui button" to="/login">Login</router-link>
<div v-if="loggedIn" v-on:click="destroyLoginToken" class="ui button">Logout</div> <!-- Hide this menu on the notes page -->
<div class="ui basic segment" v-if="this.$router.currentRoute.path != '/notes'">
<div class="ui container">
<div class="ui tabular menu">
<router-link class="item" exact-active-class="active" to="/">Home</router-link>
<router-link v-if="loggedIn" exact-active-class="active" class="item" to="/notes">Notes</router-link>
<router-link class="item" exact-active-class="active" to="/help">Help</router-link>
<router-link v-if="!loggedIn" exact-active-class="active" class="item" to="/login">Login</router-link>
<div v-if="loggedIn" v-on:click="destroyLoginToken" class="item">Logout</div>
</div>
</div>
</div> </div>
<router-view/> <router-view/>
</div> </div>
</template> </template>
@@ -28,8 +39,13 @@ export default {
//Puts token into state on page load //Puts token into state on page load
let token = localStorage.getItem('loginToken') let token = localStorage.getItem('loginToken')
let username = localStorage.getItem('username')
if(token){ if(token){
this.$store.commit('setLoginToken', token) this.$store.commit('setLoginToken', {token, username})
} else {
this.$store.commit('destroyLoginToken')
this.$router.push({'path':'/'})
} }
}, },
mounted: function(){ mounted: function(){

2
client/src/assets/semantic-helper.css
View File

@@ -9,7 +9,7 @@
.ck-content { .ck-content {
font-family: 'Open Sans' !important; font-family: 'Open Sans' !important;
font-size: 1.3rem !important; font-size: 1.3rem !important;
background: white; background-color: rgba(255, 255, 255, 0);
height: 100%; height: 100%;
overflow: hidden; overflow: hidden;
} }

122
client/src/components/HelloWorld.vue
View File

@@ -1,121 +1,13 @@
<template> <template>
<div class="hello"> <div class="ui basic segment">
<h1>{{ msg }}</h1> <div class="ui container">
<h2>Essential Links</h2> <h1>Welcome</h1>
<ul> </div>
<li> </div>
<a
href="https://vuejs.org"
target="_blank"
>
Core Docs
</a>
</li>
<li>
<a
href="https://forum.vuejs.org"
target="_blank"
>
Forum
</a>
</li>
<li>
<a
href="https://chat.vuejs.org"
target="_blank"
>
Community Chat
</a>
</li>
<li>
<a
href="https://twitter.com/vuejs"
target="_blank"
>
Twitter
</a>
</li>
<br>
<li>
<a
href="http://vuejs-templates.github.io/webpack/"
target="_blank"
>
Docs for This Template
</a>
</li>
</ul>
<h2>Ecosystem</h2>
<ul>
<li>
<a
href="http://router.vuejs.org/"
target="_blank"
>
vue-router
</a>
</li>
<li>
<a
href="http://vuex.vuejs.org/"
target="_blank"
>
vuex
</a>
</li>
<li>
<a
href="http://vue-loader.vuejs.org/"
target="_blank"
>
vue-loader
</a>
</li>
<li>
<a
href="https://github.com/vuejs/awesome-vue"
target="_blank"
>
awesome-vue
</a>
</li>
<li>
<a
href="http://marvin.local/solr"
target="_self"
>
solr
</a>
</li>
</ul>
</div>
</template> </template>
<script> <script>
export default { export default {
name: 'HelloWorld', name: 'WelcomePage'
data () {
return {
msg: 'Welcome to Your mother fucking Vue App'
}
}
} }
</script> </script>
<!-- Add "scoped" attribute to limit CSS to this component only -->
<style scoped>
h1, h2 {
font-weight: normal;
}
ul {
list-style-type: none;
padding: 0;
}
li {
display: inline-block;
margin: 0 10px;
}
a {
color: #42b983;
}
</style>

30
client/src/components/HelpPage.vue Normal file
View File

@@ -0,0 +1,30 @@
<template>
<div class="ui basic segment">
<div class="ui container">
<h2>Block formatting</h2>
<p>The following block formatting options are available:</p>
<p> Bulleted list – Start a line with * or - followed by a space.</p>
<p> Numbered list – Start a line with 1. or 1) followed by a space.</p>
<p> Headings – Start a line with # or ## or ### followed by a space to create a heading 1, heading 2 or heading 3 (up to heading 6 if options defines more headings).</p>
<p> Block quote – Start a line with > followed by a space.</p>
<h2>Inline formatting</h2>
<p>The following inline formatting options are available:<p>
<p>Bold – Type **text** or __text__,</p>
<p>Italic – Type *text* or _text_,</p>
<p>Code – Type `text`.</p>
</div>
</div>
</template>
<script>
export default {
name: 'HelpPage'
}
</script>

54
client/src/components/InputNotes.vue
View File

@@ -1,5 +1,5 @@
<template> <template>
<div id="InputNotes" class="master-note-edit" :class="[ 'position-'+position ]" @keyup.esc="close" :style="{'background-color':color}"> <div id="InputNotes" class="master-note-edit" :class="[ 'position-'+position ]" @keyup.esc="close" :style="{'background-color':color, 'color':fontColor}">
<div v-if="fancyInput == 1" class="textarea-height no-flow"> <div v-if="fancyInput == 1" class="textarea-height no-flow">
@@ -45,26 +45,6 @@
<note-tag-edit :noteId="noteid" :key="'tags-for-note-'+noteid"/> <note-tag-edit :noteId="noteid" :key="'tags-for-note-'+noteid"/>
<div class="ui segment" v-if="false">
Block formatting
The following block formatting options are available:
<p> Bulleted list – Start a line with * or - followed by a space.</p>
<p> Numbered list – Start a line with 1. or 1) followed by a space.</p>
<p> Headings – Start a line with # or ## or ### followed by a space to create a heading 1, heading 2 or heading 3 (up to heading 6 if options defines more headings).</p>
<p> Block quote – Start a line with > followed by a space.</p>
Inline formatting
The following inline formatting options are available:
Bold – Type **text** or __text__,
Italic – Type *text* or _text_,
Code – Type `text`.
</div>
</div> </div>
</template> </template>
@@ -86,12 +66,14 @@
noteText: '', noteText: '',
statusText: 'Save', statusText: 'Save',
lastNoteHash: null, lastNoteHash: null,
saveDebounce: null, //Prevent save from being called numerous times quickly
lastSaved: 0, lastSaved: 0,
updated: 'Never', updated: 'Never',
editDebounce: null, editDebounce: null,
keyPressesCounter: 0, keyPressesCounter: 0,
fancyInput: 0, //Default to basic text edit. Upgrade if set to 1 fancyInput: 0, //Default to basic text edit. Upgrade if set to 1
color: '#FFF', color: '#FFF',
fontColor: '#000',
editor: DecoupledEditor, editor: DecoupledEditor,
editorConfig: { editorConfig: {
@@ -141,9 +123,11 @@
//Grab the color of the button clicked //Grab the color of the button clicked
const style = getComputedStyle(event.target) const style = getComputedStyle(event.target)
this.color = style['background-color'] this.color = style['background-color']
this.fontColor = '#FFF'
if(this.color == "rgb(255, 255, 255)" || this.color == '#FFF'){ if(this.color == "rgb(255, 255, 255)" || this.color == '#FFF'){
this.color = null this.color = null
this.fontColor = '#000'
} }
this.lastNoteHash = 0 //Update hash to force note update on next save this.lastNoteHash = 0 //Update hash to force note update on next save
@@ -161,6 +145,13 @@
vm.noteText = response.data.text vm.noteText = response.data.text
vm.updated = response.data.updated vm.updated = response.data.updated
vm.lastNoteHash = vm.hashString(response.data.text) vm.lastNoteHash = vm.hashString(response.data.text)
vm.color = response.data.color
this.fontColor = '#FFF'
if(this.color == "rgb(255, 255, 255)" || this.color == '#FFF' || this.color == null){
this.color = null
this.fontColor = '#000'
}
if(response.data.raw_input == 1){ if(response.data.raw_input == 1){
this.fancyInput = 1 this.fancyInput = 1
@@ -241,16 +232,21 @@
} }
let vm = this let vm = this
//Only save every 1 second
clearTimeout(this.saveDebounce)
this.saveDebounce = setTimeout(() => {
//Only notify user if saving - may help with debugging in the future
vm.statusText = 'Saving'
axios.post('/api/notes/update', postData).then( response => {
vm.statusText = 'Save'
vm.updated = Math.round((+new Date)/1000)
//Only notify user if saving - may help with debugging in the future //Update last saved note hash
vm.statusText = 'Saving' vm.lastNoteHash = vm.hashString(vm.noteText)
axios.post('/api/notes/update', postData).then( response => { })
vm.statusText = 'Save' }, 500)
vm.updated = Math.round((+new Date)/1000)
//Update last saved note hash
vm.lastNoteHash = vm.hashString(vm.noteText)
})
}, },
hashString(text){ hashString(text){
var hash = 0; var hash = 0;

41
client/src/components/Login.vue
View File

@@ -1,21 +1,24 @@
<template> <template>
<div> <div class="ui container">
<h3>{{getRudeMessage}}</h3> <h3>Login</h3>
<p>Login</p> <p>Begin the login process by typing your username or email.</p>
<p>To create an account, type in the username you want to use followed by the password.</p>
<p>You will remain logged in on this browser, until you log out.</p>
<div class="ui segment" v-on:keyup.enter="submit"> <div class="ui segment" v-on:keyup.enter="submit">
<div class="field"> <div class="ui large form">
<div class="ui input"> <div class="field">
<input v-model="username" type="text" name="email" placeholder="E-mail address" autofocus> <div class="ui input">
<input v-model="username" type="text" name="email" placeholder="Username or E-mail address" autofocus>
</div>
</div> </div>
</div> <div class="field" v-if="username.length > 0">
<div class="field"> <div class="ui input">
<div class="ui input"> <input v-model="password" type="password" name="password" placeholder="Password">
<input v-model="password" type="password" name="password" placeholder="Password"> </div>
</div> </div>
<div :class="{ 'disabled':(username.length == 0 || password.length == 0)}" v-on:click="submit" class="ui massive compact fluid green submit button">Login</div>
</div> </div>
<div v-on:click="submit" class="ui teal submit button">Login</div>
</div> </div>
@@ -39,11 +42,16 @@
} }
}, },
methods: { methods: {
validate(){
console.log('Validate this bitch')
},
submit(){ submit(){
//Both fields are required
if(this.username <= 0){
return false
}
if(this.password <= 0){
return false
}
let vm = this let vm = this
let data = { let data = {
@@ -57,7 +65,8 @@
if(response.data.success){ if(response.data.success){
const token = response.data.token const token = response.data.token
vm.$store.commit('setLoginToken', token) const username = response.data.username
vm.$store.commit('setLoginToken', {token, username})
//Redirect user to notes section after login //Redirect user to notes section after login
this.$router.push('/notes') this.$router.push('/notes')

42
client/src/components/NoteTitleDisplayCard.vue
View File

@@ -2,12 +2,24 @@
<div class="ui clickable segment" @click="onClick(note.id)" :style="{'background-color':color, 'color':fontColor}"> <div class="ui clickable segment" @click="onClick(note.id)" :style="{'background-color':color, 'color':fontColor}">
<h3>{{note.text}}</h3> <h3>{{note.text}}</h3>
<p>Edited: {{$helpers.timeAgo(note.updated)}}</p> <p>Edited: {{$helpers.timeAgo(note.updated)}}</p>
<!-- Display highlights from solr results -->
<div v-if="note.note_highlights.length > 0" class="term-usage">
<p>Note Text</p>
<div class="usage-row" v-for="highlight in note.note_highlights" v-html="highlight"></div>
</div>
<div v-if="note.attachment_highlights.length > 0" class="term-usage">
<p>Note URL Text</p>
<div class="usage-row" v-for="highlight in note.attachment_highlights" v-html="highlight"></div>
</div>
<div v-if="note.tag_highlights.length > 0" class="term-usage">
Tag
<div class="ui icon large label" v-for="highlight in note.tag_highlights" v-html="highlight"></div>
</div>
</div> </div>
</template> </template>
<script> <script>
import axios from 'axios';
export default { export default {
name: 'NoteTitleDisplayCard', name: 'NoteTitleDisplayCard',
@@ -20,23 +32,29 @@
} }
}, },
beforeMount(){ beforeMount(){
this.note = this.data this.note = this.data
if(this.note.color != null && this.note.color != '#FFF'){ if(this.note.color != null && this.note.color != '#FFF'){
this.color = this.note.color this.color = this.note.color
this.fontColor = '#FFF' this.fontColor = '#FFF'
} }
},
mounted() {
},
methods: {
yup(){
}
} }
} }
</script> </script>
<style type="text/css" scoped> <style type="text/css">
.suggestion-box { .term-usage {
border: 1px solid #DDD;
padding: 10px;
}
.term-usage em {
color: green;
background-color: white;
font-weight: bold;
}
.usage-row + .usage-row {
padding: 8px 0 0;
border-top: 1px solid #DDD;
margin: 8px 0 0;
} }
</style> </style>

29
client/src/components/SearchBar.vue
View File

@@ -16,7 +16,7 @@
<div class="ui twelve wide column"> <div class="ui twelve wide column">
<div class="ui form"> <div class="ui form">
<input v-model="searchTerm" @keyup="searchKeyUp" @:keyup.enter="search" placeholder="Search Notes" /> <input v-model="searchTerm" @keyup="searchKeyUp" @:keyup.enter="search" placeholder="Search Notes" />
</div> </div>
</div> </div>
</div> </div>
@@ -33,7 +33,16 @@
<div class="ui five wide column"> <div class="ui five wide column">
<div class="ui form"> <div class="ui form">
<input v-model="searchTerm" @keyup="searchKeyUp" @:keyup.enter="search" placeholder="Search Notes" /> <input v-model="searchTerm" @keyup="searchKeyUp" @:keyup.enter="search" placeholder="Search Notes" />
</div> </div>
</div>
<div class="ui nine wide column">
<router-link class="ui basic button" to="/help">Help</router-link>
<div class="ui right floated basic button"
data-tooltip="Log Out" data-position="left center"
v-on:click="destroyLoginToken">{{username}}</div>
</div> </div>
</div> </div>
@@ -57,9 +66,10 @@
v-for="note in notes" v-for="note in notes"
:onClick="openNote" :onClick="openNote"
:data="note" :data="note"
:key="note.id + note.color" :key="note.id + note.color + searchTerm + note.note_highlights.length + note.attachment_highlights.length + note.tag_highlights.length"
/> />
</div> </div>
</div> </div>
</div> </div>
</div> </div>
@@ -83,6 +93,7 @@
}, },
data () { data () {
return { return {
username:'',
initComponent: true, initComponent: true,
commonTags: [], commonTags: [],
searchTerm: '', searchTerm: '',
@@ -95,16 +106,22 @@
activeNoteId2: null, activeNoteId2: null,
//Position determines how note is Positioned //Position determines how note is Positioned
activeNote1Position: 0, activeNote1Position: 0,
activeNote2Position: 0 activeNote2Position: 0,
} }
}, },
beforeMount(){ beforeMount(){
this.$bus.$on('close_active_note', position => { this.$bus.$on('close_active_note', position => {
this.closeNote(position) this.closeNote(position)
}) })
}, },
mounted() { mounted() {
let username = this.$store.getters.getUsername
this.username = this.ucWords(username)
this.search() this.search()
}, },
methods: { methods: {
openNote(id){ openNote(id){
@@ -171,6 +188,7 @@
vm.commonTags = response.data.tags vm.commonTags = response.data.tags
vm.notes = response.data.notes vm.notes = response.data.notes
vm.highlights = response.data.highlights
}) })
}, },
searchKeyUp(){ searchKeyUp(){
@@ -202,6 +220,9 @@
this.searchTerm = '' this.searchTerm = ''
this.searchTags = [] this.searchTags = []
this.search() this.search()
},
destroyLoginToken() {
this.$store.commit('destroyLoginToken')
} }
} }
} }

8
client/src/router/index.js
View File

@@ -4,6 +4,7 @@ import Router from 'vue-router'
import HelloWorld from '@/components/HelloWorld' import HelloWorld from '@/components/HelloWorld'
import Login from '@/components/Login' import Login from '@/components/Login'
import Notes from '@/components/Notes' import Notes from '@/components/Notes'
import HelpPage from '@/components/HelpPage'
Vue.use(Router) Vue.use(Router)
@@ -23,6 +24,11 @@ export default new Router({
path: '/notes', path: '/notes',
name: 'Notes', name: 'Notes',
component: Notes component: Notes
} },
{
path: '/help',
name: 'Help',
component: HelpPage
},
] ]
}) })

21
client/src/stores/mainStore.js
View File

@@ -8,39 +8,54 @@ export default new Vuex.Store({
state: { state: {
count: 0, count: 0,
message: 'Get out me yard ya wankers', message: 'Get out me yard ya wankers',
token: null token: null,
username: null
}, },
mutations: { mutations: {
increment (state) { increment (state) {
state.count++ state.count++
}, },
setLoginToken(state, token){ setLoginToken(state, userData){
console.log(userData)
const username = userData.username
const token = userData.token
localStorage.removeItem('loginToken') //We only want one login token per computer localStorage.removeItem('loginToken') //We only want one login token per computer
localStorage.setItem('loginToken', token) localStorage.setItem('loginToken', token)
localStorage.removeItem('username') //We only want one login token per computer
localStorage.setItem('username', username)
//Set default token to axios, every request will have header //Set default token to axios, every request will have header
axios.defaults.headers.common['Authorization'] = token axios.defaults.headers.common['Authorization'] = token
state.token = token state.token = token
state.username = username
}, },
destroyLoginToken(state){ destroyLoginToken(state){
//Remove login token from local storage and from headers //Remove login token from local storage and from headers
localStorage.removeItem('loginToken') localStorage.removeItem('loginToken')
localStorage.removeItem('username')
delete axios.defaults.headers.common['Authorization'] delete axios.defaults.headers.common['Authorization']
state.token = null state.token = null
state.username = null
} }
}, },
getters: { getters: {
getRudeMessage: state => { getRudeMessage: state => {
return state.message return state.message
}, },
getUsername: state => {
return state.username
},
getLoginToken: state => { getLoginToken: state => {
return state.token return state.token
}, },
getLoggedIn: state => { getLoggedIn: state => {
let weIn = (state.token !== null && state.token.length > 0) let weIn = (state.token !== null && state.token != undefined && state.token.length > 0)
return weIn return weIn
} }
} }

3
package.json
View File

@@ -10,7 +10,8 @@
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"body-parser": "^1.18.3", "body-parser": "^1.18.3",
"express": "^4.16.4" "express": "^4.16.4",
"request-promise": "^4.2.4"
}, },
"_moduleAliases": { "_moduleAliases": {
"@root": ".", "@root": ".",

190
server/models/Attachment.js Normal file
View File

@@ -0,0 +1,190 @@
let db = require('@config/database')
let Attachment = module.exports = {}
const cheerio = require('cheerio');
const rp = require('request-promise');
Attachment.forNote = (userId, noteId) => {
return new Promise((resolve, reject) => {
db.promise()
.query(`SELECT * FROM attachment WHERE user_id = ? AND note_id = ? AND attachment_type = 1;`, [userId, noteId])
.then((rows, fields) => {
resolve(rows[0]) //Return all tags found by query
})
.catch(console.log)
})
}
Attachment.delete = (attachmentId) => {
return new Promise((resolve, reject) => {
db.promise()
.query(`DELETE FROM attachment WHERE id = ?`, [attachmentId])
.then((rows, fields) => {
resolve(rows[0]) //Return all tags found by query
})
.catch(console.log)
})
}
Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
return new Promise((resolve, reject) => {
let solrAttachmentText = '' //Final searchable scrape text for note
if(noteText.length == 0){ resolve(solrAttachmentText) }
Attachment.forNote(userId, noteId).then(attachments => {
//Find all URLs in text
const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
let foundUrls = noteText.match(urlPattern)
//Go through each attachment, check for existing URLs
attachments.forEach(attachment => {
//URL already scraped, push text and continue
let urlIndex = foundUrls.indexOf( attachment.url )
if(urlIndex != -1){
solrAttachmentText += attachment.text
foundUrls.splice(urlIndex, 1) //Remove existing from set of found
} else {
Attachment.delete(attachment.id)
}
})
//No newly scraped URLs, resolve with looked up attachment text
if(foundUrls.length == 0){
resolve(solrAttachmentText)
}
//Process the remaining URLs into attachments
Attachment.scrapeUrlsCreateAttachments(userId, noteId, foundUrls).then( freshlyScrapedText => {
solrAttachmentText += freshlyScrapedText
resolve(solrAttachmentText)
})
})
})
}
//Return scraped text from each URL
Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
return new Promise((resolve, reject) => {
console.log('About to scrape')
console.log(foundUrls)
if(foundUrls == null || foundUrls.length == 0){resolve('')}
let processedCount = 0
let scrapedText = ''
//Process each URL passd to function, a DB entry will be created for each scrape
foundUrls.forEach(url => {
Attachment.processUrl(userId, noteId, url).then( freshlyScrapedText => {
scrapedText += freshlyScrapedText
processedCount ++
//All URLs have been scraped, return data
if(processedCount == foundUrls.length){
resolve(scrapedText)
}
})
})
})
}
Attachment.processUrl = (userId, noteId, url) => {
return new Promise((resolve, reject) => {
const excludeWords = ['share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
var removeWhitespace = /\s+/g
// console.log('Scraping ', website)
const options = {
uri: url,
transform: function (body) {
return cheerio.load(body);
}
}
rp(options).then($ => {
var desiredSearchText = ''
let pageTitle = $('title').text().replace(removeWhitespace, " ")
desiredSearchText += pageTitle + "\n"
let header = $('h1').text().replace(removeWhitespace, " ")
desiredSearchText += header + "\n"
let majorContent = ''
majorContent += $('[class*=content]').text()
.replace(removeWhitespace, " ") //Remove all whitespace
.replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,3000)
.toLowerCase()
majorContent += $('[id*=content]').text().replace(removeWhitespace, " ")
.replace(removeWhitespace, " ") //Remove all whitespace
.replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,3000) //Limit characters
.toLowerCase()
//Count frequency of each word in scraped text
let frequency = {}
majorContent.split(' ').forEach(word => {
if(excludeWords.includes(word)){
return //Exclude certain words
}
if(!frequency[word]){
frequency[word] = 0
}
frequency[word]++
})
//Create a sortable array
var sortable = [];
for (var index in frequency) {
if(frequency[index] > 1){
sortable.push([index, frequency[index]]);
}
}
//Sort them by most used words in the list
sortable.sort(function(a, b) {
return b[1] - a[1];
});
let finalWords = []
for(let i=0; i<15; i++){
if(sortable[i][0]){
finalWords.push(sortable[i][0])
}
}
desiredSearchText += finalWords.join(', ')
const created = Math.round((+new Date)/1000)
//Create attachment in DB with scrape text and provided data
db.promise()
.query(`INSERT INTO attachment
(note_id, user_id, attachment_type, text, url, last_indexed)
VALUES (?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created])
.then((rows, fields) => {
resolve(desiredSearchText) //Return found text
})
.catch(console.log)
})
})
}

223
server/models/Notes.js
View File

@@ -1,10 +1,26 @@
let db = require('@config/database') let db = require('@config/database')
let Tags = require('@models/Tags')
let Attachment = require('@models/Attachment')
var rp = require('request-promise');
var SolrNode = require('solr-node');
let Notes = module.exports = {} let Notes = module.exports = {}
// Create client
var client = new SolrNode({
host: '127.0.0.1',
port: '8983',
core: 'note',
protocol: 'http'
});
Notes.create = (userId, noteText) => { Notes.create = (userId, noteText) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if(userId == null || userId < 10){ reject('User Id required to create note') }
const created = Math.round((+new Date)/1000) const created = Math.round((+new Date)/1000)
db.promise() db.promise()
@@ -24,6 +40,33 @@ Notes.update = (userId, noteId, noteText, fancyInput, color) => {
db.promise() db.promise()
.query('UPDATE notes SET text = ?, raw_input = ?, updated = ?, color = ? WHERE id = ? AND user = ? LIMIT 1', [noteText, fancyInput, now, color, noteId, userId]) .query('UPDATE notes SET text = ?, raw_input = ?, updated = ?, color = ? WHERE id = ? AND user = ? LIMIT 1', [noteText, fancyInput, now, color, noteId, userId])
.then((rows, fields) => { .then((rows, fields) => {
//Process note text and attachment data
Attachment.scanTextForWebsites(userId, noteId, noteText).then( attachmentText => {
//
// Update Solr index
//
Tags.string(userId, noteId).then(tagString => {
// JSON Data
var data = {
'id': noteId,//string - ID of note
'user_id': userId,//int
'note_text': noteText,
'notes_tags': tagString,
'attachment_text': attachmentText,
};
// Update document to Solr server
client.update(data, function(err, result) {
if (err) { console.log(err); return; }
console.log('Note Solr Update, node/solrid ('+noteId+'):');
console.log(result.responseHeader)
});
})
})
//Send back updated response
resolve(rows[0]) resolve(rows[0])
}) })
.catch(console.log) .catch(console.log)
@@ -39,7 +82,7 @@ Notes.delete = (userId, noteId) => {
Notes.get = (userId, noteId) => { Notes.get = (userId, noteId) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
db.promise() db.promise()
.query('SELECT text, updated, raw_input FROM notes WHERE user = ? AND id = ? LIMIT 1', [userId,noteId]) .query('SELECT text, updated, raw_input, color FROM notes WHERE user = ? AND id = ? LIMIT 1', [userId,noteId])
.then((rows, fields) => { .then((rows, fields) => {
resolve(rows[0][0]) resolve(rows[0][0])
}) })
@@ -58,31 +101,36 @@ Notes.getLatest = (userId) => {
}) })
} }
Notes.solrQuery = (userId, searchQuery, searchTags) => {
return new Promise((resolve, reject) => {
if(searchQuery != '' && searchQuery != null){
let urlQuery = `/solr/note/select?hl.fl=note_text&hl=on&q=user_id:${userId} AND note_text:${searchQuery}&wt=json`
urlQuery = `/solr/note/select?
hl.fl=note_text,attachment_text,notes_tags&
hl=on&
q=user_id:${userId} AND (note_text:${searchQuery} OR attachment_text:${searchQuery} OR notes_tags:${searchQuery})&
wt=json&
fl=id&
hl.fl=note_text,attachment_text,notes_tags&
hl.snippets=20&
hl.maxAnalyzedChars=100000`
rp('http://127.0.0.1:8983'+urlQuery)
.then(function (htmlString) {
let solrResult = JSON.parse(htmlString)
resolve(solrResult)
})
} else {
resolve([])
}
})
}
Notes.search = (userId, searchQuery, searchTags) => { Notes.search = (userId, searchQuery, searchTags) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
//Default note lookup gets all notes
let noteSearchQuery = `
SELECT notes.id, SUBSTRING(text, 1, 200) as text, updated, color
FROM notes
LEFT JOIN notes_tags ON (notes.id = notes_tags.note_id)
WHERE user = ?`
let searchParams = [userId]
if(searchQuery != ''){
//If a search query is defined, search notes for that word
searchParams.push('%'+searchQuery+'%')
noteSearchQuery += ' AND text LIKE ?'
}
if(searchTags.length > 0){
//If tags are passed, use those tags in search
searchParams.push(searchTags)
noteSearchQuery += ' AND notes_tags.tag_id IN (?)'
}
//Finish up note query
noteSearchQuery += ' GROUP BY notes.id ORDER BY updated DESC, created DESC, id DESC'
//Define return data objects //Define return data objects
let returnData = { let returnData = {
@@ -90,54 +138,113 @@ Notes.search = (userId, searchQuery, searchTags) => {
'tags':[] 'tags':[]
} }
db.promise()
.query(noteSearchQuery, searchParams)
.then((noteRows, noteFields) => {
//Push all notes Notes.solrQuery(userId, searchQuery, searchTags).then( solrResult => {
returnData['notes'] = noteRows[0]
//pull out all note ids so we can fetch all tags for those notes let highlights = solrResult.highlighting
let noteIds = []
returnData['notes'].forEach(note => {
//Grab note ID for finding tags //Parse Note ID's from solr search
noteIds.push(note.id) let solrNoteIds = []
if(solrResult.response){
//Attempt to pull string out of first tag in note solrResult.response.docs.forEach(item => {
let reg = note.text.match(/<([\w]+)[^>]*>(.*?)<\/\1>/) solrNoteIds.push(parseInt(item.id))
if(reg != null){ })
note.text = reg[2] }
}
//Return all notes with HTML tags pulled out
note.text = note.text
.replace(/&[#A-Za-z0-9]+;/g,'') //Rip out all HTML entities
.replace(/<[^>]+>/g, '') //Rip out all HTML tags
})
//If no notes are returned, there are no tags, return empty //Default note lookup gets all notes
if(noteIds.length == 0){ let noteSearchQuery = `
resolve(returnData) SELECT notes.id, SUBSTRING(text, 1, 200) as text, updated, color
FROM notes
LEFT JOIN notes_tags ON (notes.id = notes_tags.note_id)
WHERE user = ?`
let searchParams = [userId]
if(solrNoteIds.length > 0){
searchParams.push(solrNoteIds)
noteSearchQuery += ' AND notes.id IN (?)'
} }
//Only show tags of selected notes // if(searchQuery != ''){
// //If a search query is defined, search notes for that word
// searchParams.push('%'+searchQuery+'%')
// noteSearchQuery += ' AND text LIKE ?'
// }
if(searchTags.length > 0){
//If tags are passed, use those tags in search
searchParams.push(searchTags)
noteSearchQuery += ' AND notes_tags.tag_id IN (?)'
}
//Finish up note query
noteSearchQuery += ' GROUP BY notes.id ORDER BY updated DESC, created DESC, id DESC'
db.promise() db.promise()
.query(`SELECT tags.id, tags.text, count(tags.id) as usages FROM notes_tags .query(noteSearchQuery, searchParams)
JOIN tags ON (tags.id = notes_tags.tag_id) .then((noteRows, noteFields) => {
WHERE notes_tags.user_id = ?
AND note_id IN (?)
GROUP BY tags.id
ORDER BY usages DESC;`,[userId, noteIds])
.then((tagRows, tagFields) => {
returnData['tags'] = tagRows[0] //Push all notes
returnData['notes'] = noteRows[0]
//pull out all note ids so we can fetch all tags for those notes
let noteIds = []
returnData['notes'].forEach(note => {
//Grab note ID for finding tags
noteIds.push(note.id)
//Attempt to pull string out of first tag in note
let reg = note.text.match(/<([\w]+)[^>]*>(.*?)<\/\1>/)
if(reg != null){
note.text = reg[2]
}
//Return all notes with HTML tags pulled out
note.text = note.text
.replace(/&[#A-Za-z0-9]+;/g,'') //Rip out all HTML entities
.replace(/<[^>]+>/g, '') //Rip out all HTML tags
note.note_highlights = []
note.attachment_highlights = []
note.tag_highlights = []
//Push in solr highlights
if(highlights && highlights[note.id] && highlights[note.id].note_text){
note['note_highlights'] = highlights[note.id].note_text
}
if(highlights && highlights[note.id] && highlights[note.id].attachment_text){
note['attachment_highlights'] = highlights[note.id].attachment_text
}
if(highlights && highlights[note.id] && highlights[note.id].notes_tags){
note['tag_highlights'] = highlights[note.id].notes_tags
}
})
//If no notes are returned, there are no tags, return empty
if(noteIds.length == 0){
resolve(returnData)
}
//Only show tags of selected notes
db.promise()
.query(`SELECT tags.id, tags.text, count(tags.id) as usages FROM notes_tags
JOIN tags ON (tags.id = notes_tags.tag_id)
WHERE notes_tags.user_id = ?
AND note_id IN (?)
GROUP BY tags.id
ORDER BY usages DESC;`,[userId, noteIds])
.then((tagRows, tagFields) => {
returnData['tags'] = tagRows[0]
resolve(returnData)
})
.catch(console.log)
resolve(returnData)
}) })
.catch(console.log) .catch(console.log)
}) })
.catch(console.log) .catch(console.log)
}) })
} }

16
server/models/Tags.js
View File

@@ -96,6 +96,22 @@ Tags.get = (userId, noteId) => {
}) })
} }
Tags.string = (userId, noteId) => {
return new Promise((resolve, reject) => {
Tags.get(userId, noteId).then(tagArray => {
let tagString = ''
tagArray.forEach( (tag, i) => {
if(i > 0){ tagString += ',' }
tagString += tag.text
})
//Output comma delimited list of tag strings
resolve(tagString)
})
})
}
Tags.lookup = (tagText) => { Tags.lookup = (tagText) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
db.promise() db.promise()

19
server/models/Users.js
View File

@@ -23,8 +23,8 @@ User.login = (username, password) => {
//User not found, create a new account with set data //User not found, create a new account with set data
if(rows[0].length == 0){ if(rows[0].length == 0){
User.create(lowerName, password) User.create(lowerName, password)
.then(result => { .then(loginToken => {
resolve(result) resolve(loginToken)
}) })
return return
} }
@@ -112,20 +112,5 @@ User.create = (username, password) => {
.catch(console.log) .catch(console.log)
})
}
//Just used for testing
User.getUsername = (userId) => {
return new Promise((resolve, reject) => {
db.promise()
.query('SELECT username FROM users WHERE id = ? LIMIT 1', [userId])
.then((rows, fields) => {
const data = rows[0][0]
resolve(data)
})
.catch(console.log)
}) })
} }

7
server/routes/user.js
View File

@@ -17,7 +17,7 @@ router.get('/about', function (req, res) {
User.getUsername(req.headers.userId) User.getUsername(req.headers.userId)
.then( data => res.send(data) ) .then( data => res.send(data) )
}) })
// define the about route // define the login route
router.post('/login', function (req, res) { router.post('/login', function (req, res) {
//Pull out variables we want //Pull out variables we want
@@ -26,7 +26,8 @@ router.post('/login', function (req, res) {
let returnData = { let returnData = {
success: false, success: false,
token: '' token: '',
username: ''
} }
User.login(username, password) User.login(username, password)
@@ -35,6 +36,8 @@ router.post('/login', function (req, res) {
//Return json web token to user //Return json web token to user
returnData['success'] = true returnData['success'] = true
returnData['token'] = loginToken returnData['token'] = loginToken
returnData['username'] = username
res.send(returnData) res.send(returnData)
}) })
.catch(e => { .catch(e => {