Searching, url indexing
* Added a help page * Cleaned up home and login pages * Menu is hidden when on notes section of app * Added username to login data * Notes now change to the color selected for the note * Note save function has a 500ms debounce to prevent spamming * Solr results now displays content from notes, tags and attachments * All note data is now indexed in solr * Notes containing URLs are now scraped and put into tag solr index * Attachments that are removed from note are deleted when url is removed * Minor little tweaks and fixes all over the place
This commit is contained in:
Max G
parent
e52ae65a42
commit
b0a8071b41
@ -1,14 +1,25 @@
|
||||
<template>
|
||||
<div id="app">
|
||||
<link href="https://fonts.googleapis.com/css?family=Open+Sans&display=swap" rel="stylesheet">
|
||||
<div>
|
||||
<router-link class="ui button" to="/">Home</router-link>
|
||||
|
||||
<router-link v-if="loggedIn" class="ui button" to="/notes">Notes</router-link>
|
||||
<router-link v-if="!loggedIn" class="ui button" to="/login">Login</router-link>
|
||||
<div v-if="loggedIn" v-on:click="destroyLoginToken" class="ui button">Logout</div>
|
||||
<link href="https://fonts.googleapis.com/css?family=Open+Sans&display=swap" rel="stylesheet">
|
||||
|
||||
<!-- Hide this menu on the notes page -->
|
||||
<div class="ui basic segment" v-if="this.$router.currentRoute.path != '/notes'">
|
||||
<div class="ui container">
|
||||
<div class="ui tabular menu">
|
||||
|
||||
<router-link class="item" exact-active-class="active" to="/">Home</router-link>
|
||||
<router-link v-if="loggedIn" exact-active-class="active" class="item" to="/notes">Notes</router-link>
|
||||
<router-link class="item" exact-active-class="active" to="/help">Help</router-link>
|
||||
<router-link v-if="!loggedIn" exact-active-class="active" class="item" to="/login">Login</router-link>
|
||||
<div v-if="loggedIn" v-on:click="destroyLoginToken" class="item">Logout</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<router-view/>
|
||||
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@ -28,8 +39,13 @@ export default {
|
||||
|
||||
//Puts token into state on page load
|
||||
let token = localStorage.getItem('loginToken')
|
||||
let username = localStorage.getItem('username')
|
||||
|
||||
if(token){
|
||||
this.$store.commit('setLoginToken', token)
|
||||
this.$store.commit('setLoginToken', {token, username})
|
||||
} else {
|
||||
this.$store.commit('destroyLoginToken')
|
||||
this.$router.push({'path':'/'})
|
||||
}
|
||||
},
|
||||
mounted: function(){
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
.ck-content {
|
||||
font-family: 'Open Sans' !important;
|
||||
font-size: 1.3rem !important;
|
||||
background: white;
|
||||
background-color: rgba(255, 255, 255, 0);
|
||||
height: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
@ -1,121 +1,13 @@
|
||||
<template>
|
||||
<div class="hello">
|
||||
<h1>{{ msg }}</h1>
|
||||
<h2>Essential Links</h2>
|
||||
<ul>
|
||||
<li>
|
||||
<a
|
||||
href="https://vuejs.org"
|
||||
target="_blank"
|
||||
>
|
||||
Core Docs
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a
|
||||
href="https://forum.vuejs.org"
|
||||
target="_blank"
|
||||
>
|
||||
Forum
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a
|
||||
href="https://chat.vuejs.org"
|
||||
target="_blank"
|
||||
>
|
||||
Community Chat
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a
|
||||
href="https://twitter.com/vuejs"
|
||||
target="_blank"
|
||||
>
|
||||
Twitter
|
||||
</a>
|
||||
</li>
|
||||
<br>
|
||||
<li>
|
||||
<a
|
||||
href="http://vuejs-templates.github.io/webpack/"
|
||||
target="_blank"
|
||||
>
|
||||
Docs for This Template
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h2>Ecosystem</h2>
|
||||
<ul>
|
||||
<li>
|
||||
<a
|
||||
href="http://router.vuejs.org/"
|
||||
target="_blank"
|
||||
>
|
||||
vue-router
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a
|
||||
href="http://vuex.vuejs.org/"
|
||||
target="_blank"
|
||||
>
|
||||
vuex
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a
|
||||
href="http://vue-loader.vuejs.org/"
|
||||
target="_blank"
|
||||
>
|
||||
vue-loader
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a
|
||||
href="https://github.com/vuejs/awesome-vue"
|
||||
target="_blank"
|
||||
>
|
||||
awesome-vue
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a
|
||||
href="http://marvin.local/solr"
|
||||
target="_self"
|
||||
>
|
||||
solr
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="ui basic segment">
|
||||
<div class="ui container">
|
||||
<h1>Welcome</h1>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
export default {
|
||||
name: 'HelloWorld',
|
||||
data () {
|
||||
return {
|
||||
msg: 'Welcome to Your mother fucking Vue App'
|
||||
}
|
||||
}
|
||||
name: 'WelcomePage'
|
||||
}
|
||||
</script>
|
||||
|
||||
<!-- Add "scoped" attribute to limit CSS to this component only -->
|
||||
<style scoped>
|
||||
h1, h2 {
|
||||
font-weight: normal;
|
||||
}
|
||||
ul {
|
||||
list-style-type: none;
|
||||
padding: 0;
|
||||
}
|
||||
li {
|
||||
display: inline-block;
|
||||
margin: 0 10px;
|
||||
}
|
||||
a {
|
||||
color: #42b983;
|
||||
}
|
||||
</style>
|
||||
|
||||
30
client/src/components/HelpPage.vue
Normal file
30
client/src/components/HelpPage.vue
Normal file
@ -0,0 +1,30 @@
|
||||
<template>
|
||||
<div class="ui basic segment">
|
||||
<div class="ui container">
|
||||
|
||||
<h2>Block formatting</h2>
|
||||
|
||||
<p>The following block formatting options are available:</p>
|
||||
|
||||
<p> Bulleted list – Start a line with * or - followed by a space.</p>
|
||||
<p> Numbered list – Start a line with 1. or 1) followed by a space.</p>
|
||||
<p> Headings – Start a line with # or ## or ### followed by a space to create a heading 1, heading 2 or heading 3 (up to heading 6 if options defines more headings).</p>
|
||||
<p> Block quote – Start a line with > followed by a space.</p>
|
||||
|
||||
<h2>Inline formatting</h2>
|
||||
|
||||
<p>The following inline formatting options are available:<p>
|
||||
|
||||
<p>Bold – Type **text** or __text__,</p>
|
||||
<p>Italic – Type *text* or _text_,</p>
|
||||
<p>Code – Type `text`.</p>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
export default {
|
||||
name: 'HelpPage'
|
||||
}
|
||||
</script>
|
||||
@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<div id="InputNotes" class="master-note-edit" :class="[ 'position-'+position ]" @keyup.esc="close" :style="{'background-color':color}">
|
||||
<div id="InputNotes" class="master-note-edit" :class="[ 'position-'+position ]" @keyup.esc="close" :style="{'background-color':color, 'color':fontColor}">
|
||||
|
||||
|
||||
<div v-if="fancyInput == 1" class="textarea-height no-flow">
|
||||
@ -45,26 +45,6 @@
|
||||
|
||||
<note-tag-edit :noteId="noteid" :key="'tags-for-note-'+noteid"/>
|
||||
|
||||
<div class="ui segment" v-if="false">
|
||||
Block formatting
|
||||
|
||||
The following block formatting options are available:
|
||||
|
||||
<p> Bulleted list – Start a line with * or - followed by a space.</p>
|
||||
<p> Numbered list – Start a line with 1. or 1) followed by a space.</p>
|
||||
<p> Headings – Start a line with # or ## or ### followed by a space to create a heading 1, heading 2 or heading 3 (up to heading 6 if options defines more headings).</p>
|
||||
<p> Block quote – Start a line with > followed by a space.</p>
|
||||
|
||||
Inline formatting
|
||||
|
||||
The following inline formatting options are available:
|
||||
|
||||
Bold – Type **text** or __text__,
|
||||
Italic – Type *text* or _text_,
|
||||
Code – Type `text`.
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@ -86,12 +66,14 @@
|
||||
noteText: '',
|
||||
statusText: 'Save',
|
||||
lastNoteHash: null,
|
||||
saveDebounce: null, //Prevent save from being called numerous times quickly
|
||||
lastSaved: 0,
|
||||
updated: 'Never',
|
||||
editDebounce: null,
|
||||
keyPressesCounter: 0,
|
||||
fancyInput: 0, //Default to basic text edit. Upgrade if set to 1
|
||||
color: '#FFF',
|
||||
fontColor: '#000',
|
||||
|
||||
editor: DecoupledEditor,
|
||||
editorConfig: {
|
||||
@ -141,9 +123,11 @@
|
||||
//Grab the color of the button clicked
|
||||
const style = getComputedStyle(event.target)
|
||||
this.color = style['background-color']
|
||||
this.fontColor = '#FFF'
|
||||
|
||||
if(this.color == "rgb(255, 255, 255)" || this.color == '#FFF'){
|
||||
this.color = null
|
||||
this.fontColor = '#000'
|
||||
}
|
||||
|
||||
this.lastNoteHash = 0 //Update hash to force note update on next save
|
||||
@ -161,6 +145,13 @@
|
||||
vm.noteText = response.data.text
|
||||
vm.updated = response.data.updated
|
||||
vm.lastNoteHash = vm.hashString(response.data.text)
|
||||
vm.color = response.data.color
|
||||
|
||||
this.fontColor = '#FFF'
|
||||
if(this.color == "rgb(255, 255, 255)" || this.color == '#FFF' || this.color == null){
|
||||
this.color = null
|
||||
this.fontColor = '#000'
|
||||
}
|
||||
|
||||
if(response.data.raw_input == 1){
|
||||
this.fancyInput = 1
|
||||
@ -241,16 +232,21 @@
|
||||
}
|
||||
|
||||
let vm = this
|
||||
//Only save every 1 second
|
||||
clearTimeout(this.saveDebounce)
|
||||
this.saveDebounce = setTimeout(() => {
|
||||
//Only notify user if saving - may help with debugging in the future
|
||||
vm.statusText = 'Saving'
|
||||
axios.post('/api/notes/update', postData).then( response => {
|
||||
vm.statusText = 'Save'
|
||||
vm.updated = Math.round((+new Date)/1000)
|
||||
|
||||
//Update last saved note hash
|
||||
vm.lastNoteHash = vm.hashString(vm.noteText)
|
||||
})
|
||||
}, 500)
|
||||
|
||||
//Only notify user if saving - may help with debugging in the future
|
||||
vm.statusText = 'Saving'
|
||||
axios.post('/api/notes/update', postData).then( response => {
|
||||
vm.statusText = 'Save'
|
||||
vm.updated = Math.round((+new Date)/1000)
|
||||
|
||||
//Update last saved note hash
|
||||
vm.lastNoteHash = vm.hashString(vm.noteText)
|
||||
})
|
||||
},
|
||||
hashString(text){
|
||||
var hash = 0;
|
||||
|
||||
@ -1,21 +1,24 @@
|
||||
<template>
|
||||
<div>
|
||||
<h3>{{getRudeMessage}}</h3>
|
||||
<p>Login</p>
|
||||
|
||||
<div class="ui container">
|
||||
<h3>Login</h3>
|
||||
<p>Begin the login process by typing your username or email.</p>
|
||||
<p>To create an account, type in the username you want to use followed by the password.</p>
|
||||
<p>You will remain logged in on this browser, until you log out.</p>
|
||||
|
||||
<div class="ui segment" v-on:keyup.enter="submit">
|
||||
<div class="field">
|
||||
<div class="ui input">
|
||||
<input v-model="username" type="text" name="email" placeholder="E-mail address" autofocus>
|
||||
<div class="ui large form">
|
||||
<div class="field">
|
||||
<div class="ui input">
|
||||
<input v-model="username" type="text" name="email" placeholder="Username or E-mail address" autofocus>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="field">
|
||||
<div class="ui input">
|
||||
<input v-model="password" type="password" name="password" placeholder="Password">
|
||||
<div class="field" v-if="username.length > 0">
|
||||
<div class="ui input">
|
||||
<input v-model="password" type="password" name="password" placeholder="Password">
|
||||
</div>
|
||||
</div>
|
||||
<div :class="{ 'disabled':(username.length == 0 || password.length == 0)}" v-on:click="submit" class="ui massive compact fluid green submit button">Login</div>
|
||||
</div>
|
||||
<div v-on:click="submit" class="ui teal submit button">Login</div>
|
||||
</div>
|
||||
|
||||
|
||||
@ -39,11 +42,16 @@
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
validate(){
|
||||
console.log('Validate this bitch')
|
||||
},
|
||||
submit(){
|
||||
|
||||
//Both fields are required
|
||||
if(this.username <= 0){
|
||||
return false
|
||||
}
|
||||
if(this.password <= 0){
|
||||
return false
|
||||
}
|
||||
|
||||
let vm = this
|
||||
|
||||
let data = {
|
||||
@ -57,7 +65,8 @@
|
||||
if(response.data.success){
|
||||
|
||||
const token = response.data.token
|
||||
vm.$store.commit('setLoginToken', token)
|
||||
const username = response.data.username
|
||||
vm.$store.commit('setLoginToken', {token, username})
|
||||
|
||||
//Redirect user to notes section after login
|
||||
this.$router.push('/notes')
|
||||
|
||||
@ -2,13 +2,25 @@
|
||||
<div class="ui clickable segment" @click="onClick(note.id)" :style="{'background-color':color, 'color':fontColor}">
|
||||
<h3>{{note.text}}</h3>
|
||||
<p>Edited: {{$helpers.timeAgo(note.updated)}}</p>
|
||||
|
||||
<!-- Display highlights from solr results -->
|
||||
<div v-if="note.note_highlights.length > 0" class="term-usage">
|
||||
<p>Note Text</p>
|
||||
<div class="usage-row" v-for="highlight in note.note_highlights" v-html="highlight"></div>
|
||||
</div>
|
||||
<div v-if="note.attachment_highlights.length > 0" class="term-usage">
|
||||
<p>Note URL Text</p>
|
||||
<div class="usage-row" v-for="highlight in note.attachment_highlights" v-html="highlight"></div>
|
||||
</div>
|
||||
<div v-if="note.tag_highlights.length > 0" class="term-usage">
|
||||
Tag
|
||||
<div class="ui icon large label" v-for="highlight in note.tag_highlights" v-html="highlight"></div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
|
||||
import axios from 'axios';
|
||||
|
||||
export default {
|
||||
name: 'NoteTitleDisplayCard',
|
||||
props: [ 'onClick', 'data' ],
|
||||
@ -20,23 +32,29 @@
|
||||
}
|
||||
},
|
||||
beforeMount(){
|
||||
|
||||
this.note = this.data
|
||||
|
||||
if(this.note.color != null && this.note.color != '#FFF'){
|
||||
this.color = this.note.color
|
||||
this.fontColor = '#FFF'
|
||||
}
|
||||
},
|
||||
mounted() {
|
||||
},
|
||||
methods: {
|
||||
yup(){
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
<style type="text/css" scoped>
|
||||
.suggestion-box {
|
||||
|
||||
<style type="text/css">
|
||||
.term-usage {
|
||||
border: 1px solid #DDD;
|
||||
padding: 10px;
|
||||
}
|
||||
.term-usage em {
|
||||
color: green;
|
||||
background-color: white;
|
||||
font-weight: bold;
|
||||
}
|
||||
.usage-row + .usage-row {
|
||||
padding: 8px 0 0;
|
||||
border-top: 1px solid #DDD;
|
||||
margin: 8px 0 0;
|
||||
}
|
||||
</style>
|
||||
@ -35,6 +35,15 @@
|
||||
<input v-model="searchTerm" @keyup="searchKeyUp" @:keyup.enter="search" placeholder="Search Notes" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="ui nine wide column">
|
||||
|
||||
<router-link class="ui basic button" to="/help">Help</router-link>
|
||||
|
||||
<div class="ui right floated basic button"
|
||||
data-tooltip="Log Out" data-position="left center"
|
||||
v-on:click="destroyLoginToken">{{username}}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="ui row">
|
||||
@ -57,9 +66,10 @@
|
||||
v-for="note in notes"
|
||||
:onClick="openNote"
|
||||
:data="note"
|
||||
:key="note.id + note.color"
|
||||
:key="note.id + note.color + searchTerm + note.note_highlights.length + note.attachment_highlights.length + note.tag_highlights.length"
|
||||
/>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -83,6 +93,7 @@
|
||||
},
|
||||
data () {
|
||||
return {
|
||||
username:'',
|
||||
initComponent: true,
|
||||
commonTags: [],
|
||||
searchTerm: '',
|
||||
@ -95,16 +106,22 @@
|
||||
activeNoteId2: null,
|
||||
//Position determines how note is Positioned
|
||||
activeNote1Position: 0,
|
||||
activeNote2Position: 0
|
||||
activeNote2Position: 0,
|
||||
}
|
||||
},
|
||||
beforeMount(){
|
||||
this.$bus.$on('close_active_note', position => {
|
||||
this.closeNote(position)
|
||||
})
|
||||
|
||||
|
||||
},
|
||||
mounted() {
|
||||
let username = this.$store.getters.getUsername
|
||||
this.username = this.ucWords(username)
|
||||
|
||||
this.search()
|
||||
|
||||
},
|
||||
methods: {
|
||||
openNote(id){
|
||||
@ -171,6 +188,7 @@
|
||||
|
||||
vm.commonTags = response.data.tags
|
||||
vm.notes = response.data.notes
|
||||
vm.highlights = response.data.highlights
|
||||
})
|
||||
},
|
||||
searchKeyUp(){
|
||||
@ -202,6 +220,9 @@
|
||||
this.searchTerm = ''
|
||||
this.searchTags = []
|
||||
this.search()
|
||||
},
|
||||
destroyLoginToken() {
|
||||
this.$store.commit('destroyLoginToken')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import Router from 'vue-router'
|
||||
import HelloWorld from '@/components/HelloWorld'
|
||||
import Login from '@/components/Login'
|
||||
import Notes from '@/components/Notes'
|
||||
import HelpPage from '@/components/HelpPage'
|
||||
|
||||
Vue.use(Router)
|
||||
|
||||
@ -23,6 +24,11 @@ export default new Router({
|
||||
path: '/notes',
|
||||
name: 'Notes',
|
||||
component: Notes
|
||||
}
|
||||
},
|
||||
{
|
||||
path: '/help',
|
||||
name: 'Help',
|
||||
component: HelpPage
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
@ -8,39 +8,54 @@ export default new Vuex.Store({
|
||||
state: {
|
||||
count: 0,
|
||||
message: 'Get out me yard ya wankers',
|
||||
token: null
|
||||
token: null,
|
||||
username: null
|
||||
},
|
||||
mutations: {
|
||||
increment (state) {
|
||||
state.count++
|
||||
},
|
||||
setLoginToken(state, token){
|
||||
setLoginToken(state, userData){
|
||||
|
||||
console.log(userData)
|
||||
|
||||
const username = userData.username
|
||||
const token = userData.token
|
||||
|
||||
localStorage.removeItem('loginToken') //We only want one login token per computer
|
||||
localStorage.setItem('loginToken', token)
|
||||
|
||||
localStorage.removeItem('username') //We only want one login token per computer
|
||||
localStorage.setItem('username', username)
|
||||
|
||||
//Set default token to axios, every request will have header
|
||||
axios.defaults.headers.common['Authorization'] = token
|
||||
|
||||
state.token = token
|
||||
state.username = username
|
||||
},
|
||||
destroyLoginToken(state){
|
||||
|
||||
//Remove login token from local storage and from headers
|
||||
localStorage.removeItem('loginToken')
|
||||
localStorage.removeItem('username')
|
||||
delete axios.defaults.headers.common['Authorization']
|
||||
state.token = null
|
||||
state.username = null
|
||||
}
|
||||
},
|
||||
getters: {
|
||||
getRudeMessage: state => {
|
||||
return state.message
|
||||
},
|
||||
getUsername: state => {
|
||||
return state.username
|
||||
},
|
||||
getLoginToken: state => {
|
||||
return state.token
|
||||
},
|
||||
getLoggedIn: state => {
|
||||
let weIn = (state.token !== null && state.token.length > 0)
|
||||
let weIn = (state.token !== null && state.token != undefined && state.token.length > 0)
|
||||
return weIn
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,7 +10,8 @@
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"body-parser": "^1.18.3",
|
||||
"express": "^4.16.4"
|
||||
"express": "^4.16.4",
|
||||
"request-promise": "^4.2.4"
|
||||
},
|
||||
"_moduleAliases": {
|
||||
"@root": ".",
|
||||
|
||||
190
server/models/Attachment.js
Normal file
190
server/models/Attachment.js
Normal file
@ -0,0 +1,190 @@
|
||||
let db = require('@config/database')
|
||||
|
||||
let Attachment = module.exports = {}
|
||||
|
||||
const cheerio = require('cheerio');
|
||||
const rp = require('request-promise');
|
||||
|
||||
Attachment.forNote = (userId, noteId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.promise()
|
||||
.query(`SELECT * FROM attachment WHERE user_id = ? AND note_id = ? AND attachment_type = 1;`, [userId, noteId])
|
||||
.then((rows, fields) => {
|
||||
resolve(rows[0]) //Return all tags found by query
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
}
|
||||
|
||||
Attachment.delete = (attachmentId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.promise()
|
||||
.query(`DELETE FROM attachment WHERE id = ?`, [attachmentId])
|
||||
.then((rows, fields) => {
|
||||
resolve(rows[0]) //Return all tags found by query
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
}
|
||||
|
||||
Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
let solrAttachmentText = '' //Final searchable scrape text for note
|
||||
|
||||
if(noteText.length == 0){ resolve(solrAttachmentText) }
|
||||
|
||||
Attachment.forNote(userId, noteId).then(attachments => {
|
||||
|
||||
//Find all URLs in text
|
||||
const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
|
||||
let foundUrls = noteText.match(urlPattern)
|
||||
|
||||
//Go through each attachment, check for existing URLs
|
||||
attachments.forEach(attachment => {
|
||||
//URL already scraped, push text and continue
|
||||
let urlIndex = foundUrls.indexOf( attachment.url )
|
||||
|
||||
if(urlIndex != -1){
|
||||
solrAttachmentText += attachment.text
|
||||
foundUrls.splice(urlIndex, 1) //Remove existing from set of found
|
||||
} else {
|
||||
Attachment.delete(attachment.id)
|
||||
}
|
||||
})
|
||||
|
||||
//No newly scraped URLs, resolve with looked up attachment text
|
||||
if(foundUrls.length == 0){
|
||||
resolve(solrAttachmentText)
|
||||
}
|
||||
|
||||
//Process the remaining URLs into attachments
|
||||
Attachment.scrapeUrlsCreateAttachments(userId, noteId, foundUrls).then( freshlyScrapedText => {
|
||||
|
||||
solrAttachmentText += freshlyScrapedText
|
||||
resolve(solrAttachmentText)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
//Return scraped text from each URL
|
||||
Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
console.log('About to scrape')
|
||||
console.log(foundUrls)
|
||||
|
||||
if(foundUrls == null || foundUrls.length == 0){resolve('')}
|
||||
|
||||
let processedCount = 0
|
||||
let scrapedText = ''
|
||||
|
||||
//Process each URL passd to function, a DB entry will be created for each scrape
|
||||
foundUrls.forEach(url => {
|
||||
Attachment.processUrl(userId, noteId, url).then( freshlyScrapedText => {
|
||||
|
||||
scrapedText += freshlyScrapedText
|
||||
processedCount ++
|
||||
|
||||
//All URLs have been scraped, return data
|
||||
if(processedCount == foundUrls.length){
|
||||
resolve(scrapedText)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Attachment.processUrl = (userId, noteId, url) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
const excludeWords = ['share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
|
||||
'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
|
||||
'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
|
||||
'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
|
||||
'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
|
||||
'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
|
||||
|
||||
var removeWhitespace = /\s+/g
|
||||
|
||||
// console.log('Scraping ', website)
|
||||
const options = {
|
||||
uri: url,
|
||||
transform: function (body) {
|
||||
return cheerio.load(body);
|
||||
}
|
||||
}
|
||||
|
||||
rp(options).then($ => {
|
||||
|
||||
var desiredSearchText = ''
|
||||
|
||||
let pageTitle = $('title').text().replace(removeWhitespace, " ")
|
||||
desiredSearchText += pageTitle + "\n"
|
||||
|
||||
let header = $('h1').text().replace(removeWhitespace, " ")
|
||||
desiredSearchText += header + "\n"
|
||||
|
||||
let majorContent = ''
|
||||
majorContent += $('[class*=content]').text()
|
||||
.replace(removeWhitespace, " ") //Remove all whitespace
|
||||
.replace(/\W\s/g, '') //Remove all non alphanumeric characters
|
||||
.substring(0,3000)
|
||||
.toLowerCase()
|
||||
majorContent += $('[id*=content]').text().replace(removeWhitespace, " ")
|
||||
.replace(removeWhitespace, " ") //Remove all whitespace
|
||||
.replace(/\W\s/g, '') //Remove all non alphanumeric characters
|
||||
.substring(0,3000) //Limit characters
|
||||
.toLowerCase()
|
||||
|
||||
//Count frequency of each word in scraped text
|
||||
let frequency = {}
|
||||
majorContent.split(' ').forEach(word => {
|
||||
if(excludeWords.includes(word)){
|
||||
return //Exclude certain words
|
||||
}
|
||||
if(!frequency[word]){
|
||||
frequency[word] = 0
|
||||
}
|
||||
frequency[word]++
|
||||
})
|
||||
|
||||
//Create a sortable array
|
||||
var sortable = [];
|
||||
for (var index in frequency) {
|
||||
if(frequency[index] > 1){
|
||||
sortable.push([index, frequency[index]]);
|
||||
}
|
||||
}
|
||||
|
||||
//Sort them by most used words in the list
|
||||
sortable.sort(function(a, b) {
|
||||
return b[1] - a[1];
|
||||
});
|
||||
|
||||
let finalWords = []
|
||||
for(let i=0; i<15; i++){
|
||||
if(sortable[i][0]){
|
||||
finalWords.push(sortable[i][0])
|
||||
}
|
||||
}
|
||||
|
||||
desiredSearchText += finalWords.join(', ')
|
||||
|
||||
const created = Math.round((+new Date)/1000)
|
||||
|
||||
//Create attachment in DB with scrape text and provided data
|
||||
db.promise()
|
||||
.query(`INSERT INTO attachment
|
||||
(note_id, user_id, attachment_type, text, url, last_indexed)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created])
|
||||
.then((rows, fields) => {
|
||||
resolve(desiredSearchText) //Return found text
|
||||
})
|
||||
.catch(console.log)
|
||||
|
||||
})
|
||||
})
|
||||
}
|
||||
@ -1,10 +1,26 @@
|
||||
let db = require('@config/database')
|
||||
|
||||
let Tags = require('@models/Tags')
|
||||
let Attachment = require('@models/Attachment')
|
||||
|
||||
var rp = require('request-promise');
|
||||
var SolrNode = require('solr-node');
|
||||
|
||||
let Notes = module.exports = {}
|
||||
|
||||
// Create client
|
||||
var client = new SolrNode({
|
||||
host: '127.0.0.1',
|
||||
port: '8983',
|
||||
core: 'note',
|
||||
protocol: 'http'
|
||||
});
|
||||
|
||||
Notes.create = (userId, noteText) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
if(userId == null || userId < 10){ reject('User Id required to create note') }
|
||||
|
||||
const created = Math.round((+new Date)/1000)
|
||||
|
||||
db.promise()
|
||||
@ -24,6 +40,33 @@ Notes.update = (userId, noteId, noteText, fancyInput, color) => {
|
||||
db.promise()
|
||||
.query('UPDATE notes SET text = ?, raw_input = ?, updated = ?, color = ? WHERE id = ? AND user = ? LIMIT 1', [noteText, fancyInput, now, color, noteId, userId])
|
||||
.then((rows, fields) => {
|
||||
|
||||
//Process note text and attachment data
|
||||
Attachment.scanTextForWebsites(userId, noteId, noteText).then( attachmentText => {
|
||||
//
|
||||
// Update Solr index
|
||||
//
|
||||
Tags.string(userId, noteId).then(tagString => {
|
||||
// JSON Data
|
||||
var data = {
|
||||
'id': noteId,//string - ID of note
|
||||
'user_id': userId,//int
|
||||
'note_text': noteText,
|
||||
'notes_tags': tagString,
|
||||
'attachment_text': attachmentText,
|
||||
};
|
||||
// Update document to Solr server
|
||||
client.update(data, function(err, result) {
|
||||
if (err) { console.log(err); return; }
|
||||
console.log('Note Solr Update, node/solrid ('+noteId+'):');
|
||||
console.log(result.responseHeader)
|
||||
});
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
//Send back updated response
|
||||
resolve(rows[0])
|
||||
})
|
||||
.catch(console.log)
|
||||
@ -39,7 +82,7 @@ Notes.delete = (userId, noteId) => {
|
||||
Notes.get = (userId, noteId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.promise()
|
||||
.query('SELECT text, updated, raw_input FROM notes WHERE user = ? AND id = ? LIMIT 1', [userId,noteId])
|
||||
.query('SELECT text, updated, raw_input, color FROM notes WHERE user = ? AND id = ? LIMIT 1', [userId,noteId])
|
||||
.then((rows, fields) => {
|
||||
resolve(rows[0][0])
|
||||
})
|
||||
@ -58,31 +101,36 @@ Notes.getLatest = (userId) => {
|
||||
})
|
||||
}
|
||||
|
||||
Notes.solrQuery = (userId, searchQuery, searchTags) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
if(searchQuery != '' && searchQuery != null){
|
||||
let urlQuery = `/solr/note/select?hl.fl=note_text&hl=on&q=user_id:${userId} AND note_text:${searchQuery}&wt=json`
|
||||
urlQuery = `/solr/note/select?
|
||||
hl.fl=note_text,attachment_text,notes_tags&
|
||||
hl=on&
|
||||
q=user_id:${userId} AND (note_text:${searchQuery} OR attachment_text:${searchQuery} OR notes_tags:${searchQuery})&
|
||||
wt=json&
|
||||
fl=id&
|
||||
hl.fl=note_text,attachment_text,notes_tags&
|
||||
hl.snippets=20&
|
||||
hl.maxAnalyzedChars=100000`
|
||||
|
||||
rp('http://127.0.0.1:8983'+urlQuery)
|
||||
.then(function (htmlString) {
|
||||
let solrResult = JSON.parse(htmlString)
|
||||
resolve(solrResult)
|
||||
})
|
||||
} else {
|
||||
resolve([])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Notes.search = (userId, searchQuery, searchTags) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
|
||||
//Default note lookup gets all notes
|
||||
let noteSearchQuery = `
|
||||
SELECT notes.id, SUBSTRING(text, 1, 200) as text, updated, color
|
||||
FROM notes
|
||||
LEFT JOIN notes_tags ON (notes.id = notes_tags.note_id)
|
||||
WHERE user = ?`
|
||||
let searchParams = [userId]
|
||||
|
||||
if(searchQuery != ''){
|
||||
//If a search query is defined, search notes for that word
|
||||
searchParams.push('%'+searchQuery+'%')
|
||||
noteSearchQuery += ' AND text LIKE ?'
|
||||
}
|
||||
if(searchTags.length > 0){
|
||||
//If tags are passed, use those tags in search
|
||||
searchParams.push(searchTags)
|
||||
noteSearchQuery += ' AND notes_tags.tag_id IN (?)'
|
||||
}
|
||||
|
||||
//Finish up note query
|
||||
noteSearchQuery += ' GROUP BY notes.id ORDER BY updated DESC, created DESC, id DESC'
|
||||
|
||||
//Define return data objects
|
||||
let returnData = {
|
||||
@ -90,54 +138,113 @@ Notes.search = (userId, searchQuery, searchTags) => {
|
||||
'tags':[]
|
||||
}
|
||||
|
||||
db.promise()
|
||||
.query(noteSearchQuery, searchParams)
|
||||
.then((noteRows, noteFields) => {
|
||||
|
||||
//Push all notes
|
||||
returnData['notes'] = noteRows[0]
|
||||
Notes.solrQuery(userId, searchQuery, searchTags).then( solrResult => {
|
||||
|
||||
//pull out all note ids so we can fetch all tags for those notes
|
||||
let noteIds = []
|
||||
returnData['notes'].forEach(note => {
|
||||
let highlights = solrResult.highlighting
|
||||
|
||||
//Grab note ID for finding tags
|
||||
noteIds.push(note.id)
|
||||
//Parse Note ID's from solr search
|
||||
let solrNoteIds = []
|
||||
if(solrResult.response){
|
||||
solrResult.response.docs.forEach(item => {
|
||||
solrNoteIds.push(parseInt(item.id))
|
||||
})
|
||||
}
|
||||
|
||||
//Attempt to pull string out of first tag in note
|
||||
let reg = note.text.match(/<([\w]+)[^>]*>(.*?)<\/\1>/)
|
||||
if(reg != null){
|
||||
note.text = reg[2]
|
||||
}
|
||||
//Return all notes with HTML tags pulled out
|
||||
note.text = note.text
|
||||
.replace(/&[#A-Za-z0-9]+;/g,'') //Rip out all HTML entities
|
||||
.replace(/<[^>]+>/g, '') //Rip out all HTML tags
|
||||
//Default note lookup gets all notes
|
||||
let noteSearchQuery = `
|
||||
SELECT notes.id, SUBSTRING(text, 1, 200) as text, updated, color
|
||||
FROM notes
|
||||
LEFT JOIN notes_tags ON (notes.id = notes_tags.note_id)
|
||||
WHERE user = ?`
|
||||
let searchParams = [userId]
|
||||
|
||||
})
|
||||
|
||||
//If no notes are returned, there are no tags, return empty
|
||||
if(noteIds.length == 0){
|
||||
resolve(returnData)
|
||||
if(solrNoteIds.length > 0){
|
||||
searchParams.push(solrNoteIds)
|
||||
noteSearchQuery += ' AND notes.id IN (?)'
|
||||
}
|
||||
|
||||
//Only show tags of selected notes
|
||||
// if(searchQuery != ''){
|
||||
// //If a search query is defined, search notes for that word
|
||||
// searchParams.push('%'+searchQuery+'%')
|
||||
// noteSearchQuery += ' AND text LIKE ?'
|
||||
// }
|
||||
if(searchTags.length > 0){
|
||||
//If tags are passed, use those tags in search
|
||||
searchParams.push(searchTags)
|
||||
noteSearchQuery += ' AND notes_tags.tag_id IN (?)'
|
||||
}
|
||||
|
||||
//Finish up note query
|
||||
noteSearchQuery += ' GROUP BY notes.id ORDER BY updated DESC, created DESC, id DESC'
|
||||
|
||||
db.promise()
|
||||
.query(`SELECT tags.id, tags.text, count(tags.id) as usages FROM notes_tags
|
||||
JOIN tags ON (tags.id = notes_tags.tag_id)
|
||||
WHERE notes_tags.user_id = ?
|
||||
AND note_id IN (?)
|
||||
GROUP BY tags.id
|
||||
ORDER BY usages DESC;`,[userId, noteIds])
|
||||
.then((tagRows, tagFields) => {
|
||||
.query(noteSearchQuery, searchParams)
|
||||
.then((noteRows, noteFields) => {
|
||||
|
||||
returnData['tags'] = tagRows[0]
|
||||
//Push all notes
|
||||
returnData['notes'] = noteRows[0]
|
||||
|
||||
//pull out all note ids so we can fetch all tags for those notes
|
||||
let noteIds = []
|
||||
returnData['notes'].forEach(note => {
|
||||
|
||||
//Grab note ID for finding tags
|
||||
noteIds.push(note.id)
|
||||
|
||||
//Attempt to pull string out of first tag in note
|
||||
let reg = note.text.match(/<([\w]+)[^>]*>(.*?)<\/\1>/)
|
||||
if(reg != null){
|
||||
note.text = reg[2]
|
||||
}
|
||||
|
||||
//Return all notes with HTML tags pulled out
|
||||
note.text = note.text
|
||||
.replace(/&[#A-Za-z0-9]+;/g,'') //Rip out all HTML entities
|
||||
.replace(/<[^>]+>/g, '') //Rip out all HTML tags
|
||||
|
||||
note.note_highlights = []
|
||||
note.attachment_highlights = []
|
||||
note.tag_highlights = []
|
||||
|
||||
//Push in solr highlights
|
||||
if(highlights && highlights[note.id] && highlights[note.id].note_text){
|
||||
note['note_highlights'] = highlights[note.id].note_text
|
||||
}
|
||||
if(highlights && highlights[note.id] && highlights[note.id].attachment_text){
|
||||
note['attachment_highlights'] = highlights[note.id].attachment_text
|
||||
}
|
||||
if(highlights && highlights[note.id] && highlights[note.id].notes_tags){
|
||||
note['tag_highlights'] = highlights[note.id].notes_tags
|
||||
}
|
||||
})
|
||||
|
||||
//If no notes are returned, there are no tags, return empty
|
||||
if(noteIds.length == 0){
|
||||
resolve(returnData)
|
||||
}
|
||||
|
||||
//Only show tags of selected notes
|
||||
db.promise()
|
||||
.query(`SELECT tags.id, tags.text, count(tags.id) as usages FROM notes_tags
|
||||
JOIN tags ON (tags.id = notes_tags.tag_id)
|
||||
WHERE notes_tags.user_id = ?
|
||||
AND note_id IN (?)
|
||||
GROUP BY tags.id
|
||||
ORDER BY usages DESC;`,[userId, noteIds])
|
||||
.then((tagRows, tagFields) => {
|
||||
|
||||
returnData['tags'] = tagRows[0]
|
||||
|
||||
resolve(returnData)
|
||||
})
|
||||
.catch(console.log)
|
||||
|
||||
resolve(returnData)
|
||||
})
|
||||
.catch(console.log)
|
||||
|
||||
})
|
||||
.catch(console.log)
|
||||
|
||||
})
|
||||
}
|
||||
@ -96,6 +96,22 @@ Tags.get = (userId, noteId) => {
|
||||
})
|
||||
}
|
||||
|
||||
Tags.string = (userId, noteId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
Tags.get(userId, noteId).then(tagArray => {
|
||||
|
||||
let tagString = ''
|
||||
tagArray.forEach( (tag, i) => {
|
||||
if(i > 0){ tagString += ',' }
|
||||
tagString += tag.text
|
||||
})
|
||||
//Output comma delimited list of tag strings
|
||||
resolve(tagString)
|
||||
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
Tags.lookup = (tagText) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.promise()
|
||||
|
||||
@ -23,8 +23,8 @@ User.login = (username, password) => {
|
||||
//User not found, create a new account with set data
|
||||
if(rows[0].length == 0){
|
||||
User.create(lowerName, password)
|
||||
.then(result => {
|
||||
resolve(result)
|
||||
.then(loginToken => {
|
||||
resolve(loginToken)
|
||||
})
|
||||
return
|
||||
}
|
||||
@ -114,18 +114,3 @@ User.create = (username, password) => {
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
//Just used for testing
|
||||
User.getUsername = (userId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
db.promise()
|
||||
.query('SELECT username FROM users WHERE id = ? LIMIT 1', [userId])
|
||||
.then((rows, fields) => {
|
||||
const data = rows[0][0]
|
||||
|
||||
resolve(data)
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
}
|
||||
@ -17,7 +17,7 @@ router.get('/about', function (req, res) {
|
||||
User.getUsername(req.headers.userId)
|
||||
.then( data => res.send(data) )
|
||||
})
|
||||
// define the about route
|
||||
// define the login route
|
||||
router.post('/login', function (req, res) {
|
||||
|
||||
//Pull out variables we want
|
||||
@ -26,7 +26,8 @@ router.post('/login', function (req, res) {
|
||||
|
||||
let returnData = {
|
||||
success: false,
|
||||
token: ''
|
||||
token: '',
|
||||
username: ''
|
||||
}
|
||||
|
||||
User.login(username, password)
|
||||
@ -35,6 +36,8 @@ router.post('/login', function (req, res) {
|
||||
//Return json web token to user
|
||||
returnData['success'] = true
|
||||
returnData['token'] = loginToken
|
||||
returnData['username'] = username
|
||||
|
||||
res.send(returnData)
|
||||
})
|
||||
.catch(e => {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user