Gigantic Update

* Migrated manual tests to jest and started working on better coverage
* Added a bookmarklet and push key generation tool allowing URL pushing from bookmarklets
* Updated web scraping with tons of bug fixes
* Updated attachments page to handle new push links
* Aggressive note change checking, if patches get out of sync, server overwrites bad updates.
This commit is contained in:
Max 2023-10-17 19:46:14 +00:00
parent b5ef64485f
commit 276a72b4ce
25 changed files with 7442 additions and 245 deletions

View File

@ -16,6 +16,9 @@ gzip "backup-$NOW.sql"
echo "Database Backup Complete on $NOW"
# Delete all but last 8 files
ls -tp | grep -v '/$' | tail -n +9 | tr '\n' '\0' | xargs -0 rm --
##
# Restore DB
##

View File

@ -117,11 +117,16 @@
<a class="link" :href="linkUrl" target="_blank">{{linkText}}</a>
<!-- Buttons -->
<div class="ui small compact basic button" v-on:click="openNote">
<div v-if="item.note_id" class="ui small compact basic button" v-on:click="openNote">
<i class="file outline icon"></i>
Open Note
</div>
<div class="ui small compact basic button" v-on:click="openEditAttachments"
<div v-if="!item.note_id" class="ui small compact basic disabled button">
<i class="angle double up icon"></i>
Pushed from Web
</div>
<div v-if="item.note_id" class="ui small compact basic button" v-on:click="openEditAttachments"
:class="{ 'disabled':this.searchParams.noteId }">
<i class="folder open outline icon"></i>
Note Files

View File

@ -4,7 +4,7 @@
<div>
<!-- thicc form display -->
<div v-if="!thin" class="ui large form" v-on:keyup.enter="register()">
<div v-if="!thin" class="ui large form" v-on:keyup.enter="register">
<div class="field">
<div class="ui input">
<input ref="nameForm" v-model="username" type="text" name="email" placeholder="Username or E-mail">
@ -15,6 +15,11 @@
<input v-model="password" type="password" name="password" placeholder="Password">
</div>
</div>
<div class="field">
<div class="ui input">
<input v-model="password2" type="password" name="password2" placeholder="Re-type Password">
</div>
</div>
<div class="field" v-if="require2FA">
<div class="ui input">
<input v-model="authToken" ref="authForm" type="text" name="authToken" placeholder="Authorization Token">
@ -24,18 +29,11 @@
<div class="ui fluid buttons">
<div v-on:click="register()" class="ui green button" :class="{ 'disabled':(username.length == 0 || password.length == 0)}">
<div v-on:click="register" class="ui green button" :class="{ 'disabled':(username.length == 0 || password.length == 0)}">
<i class="plug icon"></i>
Sign Up
</div>
<div class="or"></div>
<div :class="{ 'disabled':(username.length == 0 || password.length == 0)}" v-on:click="login()" class="ui button">
<i class="power icon"></i>
Login
</div>
</div>
</div>
<div class="sixteen wide column">
@ -49,12 +47,12 @@
</div>
<!-- Thin form display -->
<div v-if="thin" class="ui small form" v-on:keyup.enter="login()">
<div v-if="thin" class="ui small form" v-on:keyup.enter="login">
<div v-if="!require2FA" class="field"><!-- hide this field if someone is logging in with 2FA -->
<div class="ui grid">
<div class="ui sixteen wide center aligned column">
<div v-on:click="register()" class="ui green button">
<div v-on:click="register" class="ui green button">
<i class="plug icon"></i>
Sign Up Now!
</div>
@ -87,7 +85,7 @@
</div>
</div>
<div class="field">
<div v-on:click="login()" class="ui fluid button">
<div v-on:click="login" class="ui fluid button">
<i class="power icon"></i>
Login
</div>
@ -128,6 +126,7 @@
enabled: false,
username: '',
password: '',
password2: '',
authToken: '',
require2FA: false,
}
@ -160,13 +159,21 @@
},
register(){
if( this.username.length == 0 || this.password.length == 0 ){
let error = false
if(this.$route.name == 'LoginPage'){
this.$bus.$emit('notification', 'Both a Username and Password are Required')
return
}
if( this.username.length == 0 || this.password.length == 0 || this.password2.length == 0 ){
this.$bus.$emit('notification', 'All fields are required.')
error = true
}
if( this.password !== this.password2 ){
this.$bus.$emit('notification', 'Passwords must be identical.')
error = true
}
if(error){
//Login section
this.$router.push('/login')
return

View File

@ -172,6 +172,10 @@
<span class="status-menu" v-on:click=" hash=0; save()">
<span v-if="idleNote" data-position="left center" data-tooltip="Idle: Awaiting Changes">
<i class="vertically flipped grey wifi icon"></i>
</span>
<span v-if="diffsApplied > 0">
<i class="blue wave square icon"></i>
+{{ diffsApplied }}
@ -347,6 +351,10 @@
:class="{ 'fade-me-out':sizeDown }"
v-on:click="closeButtonAction()"></div> -->
<div>
</div>
</div>
</template>
@ -404,13 +412,11 @@
pinned: 0,
archived: 0,
attachmentCount: 0,
attachments: [],
styleObject: { 'noteText':null,'noteBackground':null, 'noteIcon':null, 'iconColor':null }, //Style object. Determines colors and badges
sizeDown: false, //Used to animate close state
//Settings vars
lastVisibilityState: null,
//All the squire settings
editor: null,
usersOnNote: 0,
@ -426,6 +432,9 @@
//Diff text/sync text variables
diffTextTimeout: null,
diffsApplied: null,
idleNote: true, // If note is idle, get updates from server
idleNoteTimeout: null,
reloadNoteDebounce: null,
//Used to restore caret position
lastRange: null,
@ -486,9 +495,12 @@
this.$bus.$off('new_file_upload')
this.destroyAttachmentStyles()
this.destroyWebSockets()
document.removeEventListener('visibilitychange', this.checkForUpdatedNote)
window.removeEventListener('blur', this.windowBlurEvent)
window.removeEventListener('focus', this.windowFocusEvent)
//Obliterate squire instance
this.editor.destroy()
@ -504,7 +516,9 @@
this.forceShowLoading = true
}, 500)
document.addEventListener('visibilitychange', this.checkForUpdatedNote)
window.addEventListener('blur', this.windowBlurEvent)
window.addEventListener('focus', this.windowFocusEvent)
// this.logNoteInteraction()
//Init squire as early as possible
if(this.editor && this.editor.destroy){
@ -685,13 +699,16 @@
this.editor.addEventListener('keyup', event => {
this.onKeyup(event)
this.diffText(event)
this.logNoteInteraction()
})
this.editor.addEventListener('focus', e => {
this.logNoteInteraction()
// this.diffText(e)
})
this.editor.addEventListener('blur', e => {
this.idleNote = true
this.diffText(e)
})
@ -811,6 +828,12 @@
this.pinned = response.data.pinned
}
this.archived = response.data.archived
// Fetch attachmets if the count changed
if(response.data.attachment_count > 0){
this.getAttachments()
}
this.attachmentCount = response.data.attachment_count
return true
@ -967,6 +990,9 @@
// save editor HTML after change for future comparisons
rawNoteText = editorElement.innerHTML
// update hash on patch
this.lastNoteHash = this.hashString( rawNoteText )
this.$nextTick(() => {
return resolve(true)
})
@ -975,12 +1001,13 @@
onKeyup(event){
this.statusText = 'modified'
this.idleNote = false
//Save after x seconds
clearTimeout(this.editDebounce)
this.editDebounce = setTimeout(() => {
this.save()
}, 5 * 1000)
}, 4 * 1000)
//Save after x keystrokes
this.keyPressesCounter = (this.keyPressesCounter + 1)
@ -1013,6 +1040,7 @@
}
//tell websockets to truncate history at this save
this.lastNoteHash = currentHash //Update last saved note hash
this.$io.emit('truncate_diffs_at_save', {'rawTextId':this.rawTextId, 'hash':currentHash })
const postData = {
@ -1033,42 +1061,52 @@
this.modified = true
this.diffsApplied = 0
//Update last saved note hash
this.lastNoteHash = currentHash
return resolve(true)
})
.catch(error => { this.$bus.$emit('notification', 'Failed to Save Note') })
})
},
checkForUpdatedNote(){
loadNoteNextFromServer(){
const now = +new Date
//Only check every 3 seconds
const checkForUpdateTimeout = now - this.lastInteractionTimestamp > (2 * 1000)
clearTimeout(this.reloadNoteDebounce)
this.reloadNoteDebounce = setTimeout(() => {
//If user leaves page then returns to page, reload the first batch
if(this.lastVisibilityState == 'hidden' && document.visibilityState == 'visible' && checkForUpdateTimeout){
// flash note text to show the update
// this.setText('')
//Focus Regained on Note, check for update
axios.post('/api/note/get', { 'noteId': this.noteid })
.then(response => {
const serverTextHash = this.hashString( response.data.text )
this.setupLoadedNoteData(response)
if(this.lastNoteHash != serverTextHash){
// console.log('note was changed UPDATE THAT BITCH!!!!')
this.setupLoadedNoteData(response)
//Manually set squire text to show
this.setText(this.noteText)
//Manually set squire text to show
this.setText(this.noteText)
}
})
}, 200)
},
windowFocusEvent(){
//Only check if its been greater than a few seconds
const now = +new Date
const checkForUpdateTimeout = now - this.lastInteractionTimestamp > (3 * 1000)
//If user leaves page then returns to page, reload the first batch
if(checkForUpdateTimeout){
this.loadNoteNextFromServer()
this.lastInteractionTimestamp = now
}
//Keep track of visibility change and last interaction time
this.lastVisibilityState = document.visibilityState
},
windowBlurEvent(){
this.idleNote = true
this.lastInteractionTimestamp = +new Date
},
@ -1127,6 +1165,9 @@
this.$io.removeListener('past_diffs')
this.$io.removeListener('update_user_count')
this.$io.removeListener('incoming_diff')
this.$io.removeListener('update_note_attachments')
clearTimeout(this.idleNoteTimeout)
},
initWebsocketEvents(){
@ -1159,6 +1200,31 @@
})
})
this.$io.on('new_note_text_saved', ({noteId, hash}) => {
const sameIdCheck = (this.idleNote && this.noteid == noteId)
const differentHashCheck = (hash != this.lastNoteHash)
// if hashes do not match, reload text from server
if(sameIdCheck && differentHashCheck){
this.loadNoteNextFromServer()
}
})
this.$io.on('update_note_attachments', () => {
this.getAttachments()
})
},
logNoteInteraction(){
this.idleNote = false
clearTimeout(this.idleNoteTimeout)
this.idleNoteTimeout = setTimeout(() => {
this.idleNote = true
}, 5000)
},
saveCaretPosition(event){
@ -1202,6 +1268,88 @@
element.style.height = (element.scrollHeight) +'px'
}
},
destroyAttachmentStyles(){
// Remove attachment preview styles
var head = document.head
var styleElement = document.getElementById('attachmentGeneratedStyles'+this.noteid)
if(styleElement){
head.removeChild(styleElement)
}
},
getAttachments(){
axios.post('/api/attachment/search', {'noteId':this.noteid})
.then( results => {
// generate new style group
var style = document.createElement('style')
style.id = 'attachmentGeneratedStyles'+this.noteid
style.type = 'text/css'
// iterate attachments and build unique style for each
let attachmentStyles = []
results.data.forEach(attachment => {
// thumbnail location
const bgurl = `/api/static/thumb_${attachment.file_location}`
let padding = '2px 0 0'
// increase padding if there is a valid file
if(attachment.file_location){
padding = '13px 0 13px 155px'
}
// unescaped characters will break content attribute
const strippedText = attachment.text
.replace(/'/g, "\\'") //Escape ' s
.replace(/\n/g, '\\A') //Escape new lines
// strip down URL, *= matches anywhere is string
const substringsToRemove = ['https://','http://','www.']
var pattern = new RegExp(substringsToRemove.join('|'), 'g');
const strippedurl = attachment.url
.replace(pattern, '') // remove url protocol
.replace(/&.*/, '') // remove anything after &
const cleanStyle = `
.squire-box a[href*="${strippedurl}" i]::before {
content: '${strippedText}';
display: inline-block;
padding: ${padding};
pointer-events: none;
font-size: 1.3em !important;
width: 100%;
background-position: left center;
background-repeat: no-repeat;
background-size: 140px auto;
background-image: url(${bgurl});
border-bottom: solid 1px var(--main-accent);
margin-bottom: -10px;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.squire-box a[href*="${strippedurl}" i] {
font-size: 0.8em;
}
`
.replace(/\t|\r|\n/gm, "") // Remove tabs, new lines, returns
.replace(/\s+/g, ' ') // remove double spaces
attachmentStyles.push(cleanStyle)
})
// Destroy just before creating new to prevent page jumping
this.destroyAttachmentStyles()
// push new styles into <head>
style.innerHTML = attachmentStyles.join(' ')
document.head.appendChild(style)
})
.catch(error => { console.log(error);this.$bus.$emit('notification', 'Failed to Search Attachments') })
},
}
}
</script>

View File

@ -1,7 +1,7 @@
<template>
<div>
<div class="button-fix">
<div class="ui right floated basic shrinking icon button" v-on:click="showPasteInputArea">
<i class="paste icon"></i>
<i class="green paste icon"></i>
Paste
</div>
<div class="shade" v-if="showPasteArea" @click.prevent="close">

View File

@ -8,6 +8,13 @@
<div class="content">
Files
<div class="sub header">Uploaded Files and Websites from notes.</div>
<div class="sub header">
<i class="green angle double up icon icon"></i>
<router-link
to="/bookmarklet">
Push any website to solid scribe
</router-link>
</div>
</div>
</h2>
@ -125,6 +132,11 @@
//Load more attachments on scroll
window.addEventListener('scroll', this.onScroll)
this.$io.on('update_note_attachments', () => {
this.reset()
this.searchAttachments()
})
//Mount notes on load if note ID is set
this.searchAttachments()
},
@ -132,6 +144,8 @@
//Remove scroll event on destroy
window.removeEventListener('scroll', this.onScroll)
this.$io.removeListener('update_note_attachments')
},
watch:{
$route (to, from){

View File

@ -0,0 +1,66 @@
<template>
<div class="text-container squire-box">
<h2 class="ui header">
<i class="green angle double up icon icon"></i>
<div class="content">
Push URL to Solid Scribe - Bookmarklet
<div class="sub header">Push any website to your file list.</div>
</div>
</h2>
<p>A bookmarklet is a small piece of code that can be run from a bookmark.</p>
<p>Use the bookmarklet below to push URLs of website to solid scribe for later</p>
<p>The bookmarklet works in a secure way and won't leak any data.</p>
<p>To install the bookmarklet, all you need to do is drag it to your bookmarks bar.</p>
<h2>
Drag the link below to your bookmarks.
</h2>
<h3>
<a :href="`${(bookmarkletscript)}`" class="ui huge text">Push to SolidScribe</a>
</h3>
</div>
</template>
<script>
import axios from 'axios'
export default {
components: {
},
data: function(){
return {
loading: true,
bookmarkletscript:'',
}
},
beforeCreate: function(){
// Perform Login check
this.$parent.loginGateway()
},
mounted: function(){
this.getBookmarklet()
},
beforeDestroy(){
},
methods: {
getBookmarklet(){
this.loading = true
axios.post('/api/attachment/getbookmarklet')
.then( results => {
this.bookmarkletscript = results.data
})
.catch(error => { this.$bus.$emit('notification', 'Failed to get bookmarklet') })
},
}
}
</script>

View File

@ -13,6 +13,7 @@ const NotesPage = () => import(/* webpackChunkName: "NotesPage" */ '@/pages/Note
const QuickPage = () => import(/* webpackChunkName: "QuickPage" */ '@/pages/QuickPage')
const AttachmentsPage = () => import(/* webpackChunkName: "AttachmentsPage" */ '@/pages/AttachmentsPage')
const OverviewPage = () => import(/* webpackChunkName: "OverviewPage" */ '@/pages/OverviewPage')
const BookmarkletPage = () => import(/* webpackChunkName: "BookmarkletPage" */ '@/pages/BookmarkletPage')
const NotFoundPage = () => import(/* webpackChunkName: "404Page" */ '@/pages/NotFoundPage')
Vue.use(Router)
@ -67,6 +68,12 @@ export default new Router({
meta: {title:'Terms'},
component: TermsPage
},
{
path: '/bookmarklet',
name: 'Bookmarklet',
meta: {title:'Bookmarklet'},
component: BookmarkletPage
},
{
path: '/settings',
name: 'Settings',

11
jest.config.js Normal file
View File

@ -0,0 +1,11 @@
const path = '../../'
const prefix = '/$1'
module.exports = {
moduleNameMapper: {
"@root/(.*)": ".",
"@models/(.*)": path+"server/models"+prefix,
"@routes/(.*)": path+"server/routes"+prefix,
"@helpers/(.*)": path+"server/helpers"+prefix,
"@config/(.*)": path+"server/config"+prefix,
}
}

6486
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,10 +1,10 @@
{
"name": "personal-internet",
"version": "1.0.0",
"description": "Personal or Private net",
"description": "Encrypted note taking applications",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
"test": "jest"
},
"author": "Max",
"license": "ISC",
@ -33,5 +33,8 @@
"@routes": "server/routes",
"@helpers": "server/helpers",
"@config": "server/config"
},
"devDependencies": {
"jest": "^29.7.0"
}
}

View File

@ -1,5 +1,7 @@
//Import mysql2 package
const mysql = require('mysql2');
const os = require('os') //Used to get path of home directory
const result = require('dotenv').config({ path:(os.homedir()+'/.env') })
// Create the connection pool.
const pool = mysql.createPool({

View File

@ -72,6 +72,8 @@ CryptoString.createSalt = () => {
return crypto.randomBytes(SALT_BYTE_SIZE).toString('base64')
}
// Creates a small random salt
CryptoString.createSmallSalt = () => {
return crypto.randomBytes(20).toString('base64')

View File

@ -6,7 +6,7 @@ let SiteScrape = module.exports = {}
const removeWhitespace = /\s+/g
const commonWords = ['share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
const commonWords = ['just','start','what','these','how', 'was', 'being','can','way','share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
@ -162,19 +162,28 @@ SiteScrape.getKeywords = ($) => {
majorContent += $('[class*=content]').text()
.replace(removeWhitespace, " ") //Remove all whitespace
.replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,3000) //Limit to 3000 characters
// .replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,6000) //Limit to 6000 characters
.toLowerCase()
.replace(/[^A-Za-z0-9- ]/g, '');
console.log(majorContent)
//Count frequency of each word in scraped text
let frequency = {}
majorContent.split(' ').forEach(word => {
if(commonWords.includes(word)){
return //Exclude certain words
// Exclude short or common words
if(commonWords.includes(word) || word.length <= 2){
return
}
if(!frequency[word]){
frequency[word] = 0
}
// Skip some plurals
if(frequency[word+'s'] || frequency[word+'es']){
return
}
frequency[word]++
})
@ -192,7 +201,7 @@ SiteScrape.getKeywords = ($) => {
});
let finalWords = []
for(let i=0; i<5; i++){
for(let i=0; i<6; i++){
if(sortable[i] && sortable[i][0]){
finalWords.push(sortable[i][0])
}

View File

@ -20,6 +20,8 @@ const helmet = require('helmet')
const express = require('express')
const app = express()
app.use( helmet() )
// allow for the parsing of url encoded forms
app.use(express.urlencoded({ extended: true }));
//
@ -199,7 +201,7 @@ io.on('connection', function(socket){
http.listen(ports.socketIo, function(){
console.log(`Socke.io: Listening on port ${ports.socketIo}!`)
console.log(`Socke.io: Listening on port ${ports.socketIo}`)
});
//Enable json body parsing in requests. Allows me to post data in ajax calls
@ -240,17 +242,21 @@ app.use(function(req, res, next){
// Test Area
const printResults = true
let UserTest = require('@models/User')
let NoteTest = require('@models/Note')
let AuthTest = require('@helpers/Auth')
Auth.test()
UserTest.keyPairTest('genMan30', '1', printResults)
.then( ({testUserId, masterKey}) => NoteTest.test(testUserId, masterKey, printResults))
.then( message => {
if(printResults) console.log(message)
Auth.testTwoFactor()
})
// const printResults = true
// let UserTest = require('@models/User')
// let NoteTest = require('@models/Note')
// let AuthTest = require('@helpers/Auth')
// Auth.test()
// UserTest.keyPairTest('genMan30', '1', printResults)
// .then( ({testUserId, masterKey}) =>
// NoteTest.test(testUserId, masterKey, printResults))
// .then( message => {
// if(printResults) console.log(message)
// Auth.testTwoFactor()
// })
// .catch((error) => {
// console.log(error)
// })
//Test
app.get('/api', (req, res) => res.send('Solidscribe /API is up and running'))

View File

@ -1,6 +1,7 @@
let db = require('@config/database')
let SiteScrape = require('@helpers/SiteScrape')
const cs = require('@helpers/CryptoString')
let Attachment = module.exports = {}
@ -47,13 +48,15 @@ Attachment.textSearch = (userId, searchTerm) => {
}
Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeShared) => {
console.log([userId, noteId, attachmentType, offset, setSize, includeShared])
return new Promise((resolve, reject) => {
let params = [userId]
let query = `
SELECT attachment.*, note.share_user_id FROM attachment
JOIN note ON (attachment.note_id = note.id)
WHERE attachment.user_id = ? AND visible = 1 `
LEFT JOIN note ON (attachment.note_id = note.id)
WHERE attachment.user_id = ? AND visible = 1
`
if(noteId && noteId > 0){
//
@ -76,6 +79,11 @@ Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeSha
query += `AND note.archived = ${ attachmentType == 'archived' ? '1':'0' } `
query += `AND note.trashed = ${ attachmentType == 'trashed' ? '1':'0' } `
if(!attachmentType){
// Null note ID means it was pushed by bookmarklet
query += 'OR attachment.note_id IS NULL '
}
}
@ -102,18 +110,6 @@ Attachment.search = (userId, noteId, attachmentType, offset, setSize, includeSha
})
}
//Returns all attachments
Attachment.forNote = (userId, noteId) => {
return new Promise((resolve, reject) => {
db.promise()
.query(`SELECT * FROM attachment WHERE user_id = ? AND note_id = ? AND visible = 1 ORDER BY last_indexed DESC;`, [userId, noteId])
.then((rows, fields) => {
resolve(rows[0]) //Return all attachments found by query
})
.catch(console.log)
})
}
Attachment.urlForNote = (userId, noteId) => {
return new Promise((resolve, reject) => {
db.promise()
@ -189,6 +185,7 @@ Attachment.delete = (userId, attachmentId, urlDelete = false) => {
.catch(console.log)
}
})
.catch(console.log)
})
}
@ -305,9 +302,13 @@ Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
//Once everything is done being scraped, emit new attachment events
SocketIo.to(userId).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(userId).emit('update_note_attachments')
solrAttachmentText += freshlyScrapedText
resolve(solrAttachmentText)
})
.catch(console.log)
})
})
}
@ -335,9 +336,13 @@ Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
//All URLs have been scraped, return data
if(processedCount == foundUrls.length){
resolve(scrapedText)
console.log('All urls scraped')
return resolve(scrapedText)
}
})
.catch(error => {
console.log('Site Scrape error', error)
})
})
})
}
@ -347,8 +352,8 @@ Attachment.downloadFileFromUrl = (url) => {
return new Promise((resolve, reject) => {
if(url == null || url == undefined || url == ''){
resolve(null)
if(!url){
return resolve(null)
}
const random = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
@ -356,8 +361,7 @@ Attachment.downloadFileFromUrl = (url) => {
let fileName = random+'_scrape'
let thumbPath = 'thumb_'+fileName
console.log('Scraping image url')
console.log(url)
console.log('Scraping image url', url)
console.log('Getting ready to scrape ', url)
@ -395,7 +399,7 @@ Attachment.downloadFileFromUrl = (url) => {
Attachment.processUrl = (userId, noteId, url) => {
const scrapeTime = 20*1000;
const scrapeTime = 5*1000;
return new Promise((resolve, reject) => {
@ -450,9 +454,10 @@ Attachment.processUrl = (userId, noteId, url) => {
var desiredSearchText = ''
desiredSearchText += pageTitle
if(keywords){
desiredSearchText += "\n" + keywords
desiredSearchText += "\n " + keywords
}
console.log('Results from site scrape-------------')
console.log({
pageTitle,
hostname,
@ -502,48 +507,142 @@ Attachment.processUrl = (userId, noteId, url) => {
})
.catch(error => {
// console.log('Scrape pooped out')
// console.log('Issue with scrape')
console.log(error)
// resolve('')
console.log('Scrape pooped out')
console.log('Issue with scrape', error.statusCode)
clearTimeout(requestTimeout)
return resolve('No site text')
})
requestTimeout = setTimeout( () => {
console.log('Cancel the request, its taking to long.')
request.cancel()
desiredSearchText = 'No Description for -> '+url
created = Math.round((+new Date)/1000)
db.promise()
.query(`UPDATE attachment SET
text = ?,
last_indexed = ?,
WHERE id = ?
`, [desiredSearchText, created, insertedId])
.then((rows, fields) => {
resolve(desiredSearchText) //Return found text
})
.catch(console.log)
//Create attachment in DB with scrape text and provided data
// db.promise()
// .query(`INSERT INTO attachment
// (note_id, user_id, attachment_type, text, url, last_indexed)
// VALUES (?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created])
// .then((rows, fields) => {
// resolve(desiredSearchText) //Return found text
// })
// .catch(console.log)
return resolve('Request Timeout')
}, scrapeTime )
})
}
Attachment.generatePushKey = (userId) => {}
Attachment.generatePushKey = (userId) => {
return new Promise((resolve, reject) => {
Attachment.deletePushKey = (userId) => {}
db.promise()
.query("SELECT pushkey FROM user WHERE id = ? LIMIT 1", [userId])
.then((rows, fields) => {
Attachment.getPushkey = (userId) => {}
const pushKey = rows[0][0].pushkey
Attachment.pushUrl = (userId) => {}
// push key exists
if(pushKey && pushKey.length > 0){
return resolve(pushKey)
} else {
// generate and save a new key
const newPushKey = cs.createSmallSalt()
db.promise()
.query('UPDATE user SET pushkey = ? WHERE id = ? LIMIT 1', [newPushKey,userId])
.then((rows, fields) => {
return resolve(newPushKey)
})
}
})
})
}
Attachment.deletePushKey = (userId) => {
return new Promise((resolve, reject) => {
db.promise()
.query('UPDATE user SET pushkey = null WHERE id = ? LIMIT 1', [userId])
.then((rows, fields) => {
return resolve(rows[0].affectedRows == 1)
})
})
}
Attachment.getPushkeyBookmarklet = (userId) => {
return new Promise((resolve, reject) => {
Attachment.generatePushKey(userId)
.then( pushKey => {
let bookmarklet = Attachment.generateBookmarkletText(pushKey)
return resolve(bookmarklet)
})
})
}
Attachment.pushUrl = (pushkey,url) => {
return new Promise((resolve, reject) => {
let userId = null
pushkey = pushkey.replace(/ /g, '+')
db.promise()
.query("SELECT id FROM user WHERE pushkey = ? LIMIT 1", [pushkey])
.then((rows, fields) => {
if(rows[0].length == 0){
return resolve(true)
}
userId = rows[0][0].id
return Attachment.scrapeUrlsCreateAttachments(userId, null, [url])
})
.then(() => {
if(typeof SocketIo != 'undefined'){
//Once everything is done being scraped, emit new attachment events
SocketIo.to(userId).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(userId).emit('update_note_attachments')
}
return resolve(true)
})
.catch(console.log)
})
}
Attachment.generateBookmarkletText = (pushKey) => {
const endpoint = '/api/public/pushmebaby'
let url = 'https://www.solidscribe.com' + endpoint
if(process.env.NODE_ENV === 'development'){
// url = 'https://192.168.1.164' + endpoint
}
// Terminate each line with a semi-colon, super important, since spaces are removed.
// document.getElementById(id).remove();
url += '?pushkey='+encodeURIComponent(pushKey)
const bookmarkletV3 = `
javascript: (() => {
var p = encodeURIComponent(window.location.href);
var n = "`+url+`&url="+p;
window.open(n, '_blank', 'noopener=noopener');
window.focus();
var k = document.createElement("div");
k.setAttribute("style", "position:fixed;right:10px;top:10px;z-index:222222;border-radius:4px;font-size:1.3em;padding:20px 15px;background: #8f51be;color:white;");
k.innerHTML = "Posted URL to your Solid Scribe account";
document.body.appendChild(k);
setTimeout(()=>{
k.remove();
},5000);
})();
`
return bookmarkletV3
.replace(/\t|\r|\n/gm, "") // Remove tabs, new lines, returns
.replace(/\s+/g, ' ') // remove double spaces
.trim()
}

View File

@ -17,6 +17,7 @@ const fs = require('fs')
const gm = require('gm')
Note.test = (userId, masterKey, printResults) => {
return false;
return new Promise((resolve, reject) => {
@ -162,6 +163,10 @@ Note.test = (userId, masterKey, printResults) => {
return resolve('Test: Complete ---')
})
.catch(error => {
console.log(error)
return reject(error)
})
})
}
@ -193,7 +198,7 @@ Note.create = (userId, noteTitle = '', noteText = '', masterKey) => {
})
.then((rows, fields) => {
if(SocketIo){
if(typeof SocketIo != 'undefined'){
SocketIo.to(userId).emit('new_note_created', rows[0].insertId)
}
@ -341,7 +346,7 @@ Note.reindex = (userId, masterKey, removeId = null) => {
setTimeout(() => {
if(masterKey == null || note.salt == null){
console.log('Error indexing note', note.id)
console.log('Error indexing note - master key or salt missing', note.id)
return resolve(true)
}
@ -390,13 +395,13 @@ Note.reindex = (userId, masterKey, removeId = null) => {
return Promise.all(reindexQueue)
})
.then(rawSearchIndex => {
.then(updatePromiseResults => {
const created = Math.round((+new Date)/1000)
const jsonSearchIndex = JSON.stringify(searchIndex)
const encryptedJsonIndex = cs.encrypt(masterKey, searchIndexSalt, jsonSearchIndex)
return db.promise().query("UPDATE user_encrypted_search_index SET `index` = ?, `last_update` = ? WHERE (`user_id` = ?) LIMIT 1",
db.promise().query("UPDATE user_encrypted_search_index SET `index` = ?, `last_update` = ? WHERE (`user_id` = ?) LIMIT 1",
[encryptedJsonIndex, created, userId])
.then((rows, fields) => {
@ -406,6 +411,7 @@ Note.reindex = (userId, masterKey, removeId = null) => {
.then((rows, fields) => {
// console.log('Indexd Note Count: ' + rows[0]['affectedRows'])
// @TODO - Return number of reindexed notes
resolve(true)
})
@ -507,12 +513,12 @@ Note.update = (userId, noteId, noteText, noteTitle, color, pinned, archived, has
})
.then((rows, fields) => {
if(SocketIo){
if(typeof SocketIo != 'undefined'){
SocketIo.to(userId).emit('new_note_text_saved', {noteId, hash})
}
//Async attachment reindex
Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
//Async attachment reindex
Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
}
//Send back updated response
resolve(rows[0])
@ -739,12 +745,13 @@ Note.get = (userId, noteId, masterKey) => {
const nowTime = Math.round((+new Date)/1000)
db.promise().query(`UPDATE note SET opened = ? WHERE (id = ?)`, [nowTime, noteId])
//Return note data
// delete noteData.salt //remove salt from return data
// delete noteData.encrypted_share_password_key
noteData.lockedOut = noteLockedOut
resolve(noteData)
.then(results => {
//Return note data
// delete noteData.salt //remove salt from return data
// delete noteData.encrypted_share_password_key
noteData.lockedOut = noteLockedOut
resolve(noteData)
})
})
.catch(error => {

View File

@ -9,7 +9,8 @@ const speakeasy = require('speakeasy')
let User = module.exports = {}
const version = '3.6.3'
const version = '3.8.0'
// 3.7.3 - diff/patch update
//Login a user, if that user does not exist create them
//Issues login token
@ -552,6 +553,12 @@ User.revokeActiveSessions = (userId, sessionId) => {
User.deleteUser = (userId, password) => {
if(!userId || !password){
return new Promise((resolve, reject) => {
return resolve('Missing User ID or Password. No Action Taken.')
})
}
//Verify user is correct by decryptig master key with password
let deletePromises = []
@ -584,77 +591,3 @@ User.deleteUser = (userId, password) => {
return Promise.all(deletePromises)
}
User.keyPairTest = (testUserName = 'genMan', password = '1', printResults) => {
return new Promise((resolve, reject) => {
let masterKey = null
let testUserId = null
const randomUsername = Math.random().toString(36).substring(2, 15);
const randomPassword = '1'
const secondPassword = '2'
User.register(testUserName, password)
.then( ({ token, userId }) => {
testUserId = userId
if(printResults) console.log('Test: Register User '+testUserName+' - Pass')
return User.getMasterKey(testUserId, password)
})
.then(newMasterKey => {
masterKey = newMasterKey
if(printResults) console.log('Test: Generate/Decrypt Master Key - Pass')
return User.generateKeypair(testUserId, masterKey)
})
.then(({publicKey, privateKey}) => {
const publicKeyMessage = 'Test: Public key decrypt - Pass'
const privateKeyMessage = 'Test: Private key decrypt - Pass'
//Encrypt Message with private Key
const privateKeyEncrypted = crypto.privateEncrypt(privateKey, Buffer.from(privateKeyMessage, 'utf8')).toString('base64')
const decryptedPrivate = crypto.publicDecrypt(publicKey, Buffer.from(privateKeyEncrypted, 'base64'))
//Conver back to a string
if(printResults) console.log(decryptedPrivate.toString('utf8'))
//Encrypt with public key
const pubEncrMsc = crypto.publicEncrypt(publicKey, Buffer.from(publicKeyMessage, 'utf8')).toString('base64')
const publicDeccryptMessage = crypto.privateDecrypt(privateKey, Buffer.from(pubEncrMsc, 'base64') )
//Convert it back to string
if(printResults) console.log(publicDeccryptMessage.toString('utf8'))
return User.login(testUserName, password)
})
.then( ({token, userId}) => {
if(printResults) console.log('Test: Login New User - Pass')
return User.changePassword(testUserId, randomPassword, secondPassword)
})
.then(passwordChangeResults => {
if(printResults) console.log('Test: Password Change - ', passwordChangeResults?'Pass':'Fail')
return User.login(testUserName, secondPassword)
})
.then(reLogin => {
if(printResults) console.log('Test: Login With new Password - Pass')
return User.getMasterKey(testUserId, secondPassword)
})
.then(newMasterKey => {
masterKey = newMasterKey
resolve({testUserId, masterKey})
})
})
}

View File

@ -35,11 +35,6 @@ router.post('/textsearch', function (req, res) {
.then( data => res.send(data) )
})
router.post('/get', function (req, res) {
Attachment.forNote(userId, req.body.noteId)
.then( data => res.send(data) )
})
router.post('/update', function (req, res) {
Attachment.update(userId, req.body.attachmentId, req.body.updatedText, req.body.noteId)
.then( result => {
@ -67,12 +62,13 @@ router.post('/upload', upload.single('file'), function (req, res, next) {
//
// Push URL to attachments
// push action on - public controller
//
// get push key
router.get('/getpushkey', function (req, res) {
router.post('/getbookmarklet', function (req, res) {
Attachment.delete(userId, req.body.attachmentId)
Attachment.getPushkeyBookmarklet(userId)
.then( data => res.send(data) )
})
@ -86,9 +82,4 @@ router.post('/deletepushkey', function (req, res) {
})
// push url to attchments
router.get('/pushurl', function (req, res) {
})
module.exports = router

View File

@ -4,6 +4,7 @@ const rateLimit = require('express-rate-limit')
const Note = require('@models/Note')
const User = require('@models/User')
const Attachment = require('@models/Attachment')
@ -56,6 +57,29 @@ router.post('/register', registerLimiter, function (req, res) {
})
})
//
// Public Pushme Action
//
const pushMeLimiter = rateLimit({
windowMs: 30 * 60 * 1000, //30 min window
max: 50, // start blocking after x requests
message:'Error'
})
router.get('/pushmebaby', pushMeLimiter, function (req, res) {
Attachment.pushUrl(req.query.pushkey, req.query.url)
.then((() => {
const jsCode = `
<script>
window.close();
</script>
<h1>Posting URL</h1>
`;
res.header('Content-Security-Policy', "script-src 'unsafe-inline'");
res.set('Content-Type', 'text/html');
res.send(Buffer.from(jsCode));
}))
})
module.exports = router

View File

@ -0,0 +1,100 @@
const Attachment = require('../../models/Attachment')
const User = require('../../models/User')
const testUserName = 'jestTestUserAttachment'
const password = 'Beans19934!!!'
let newUserId = null
let masterKey = null
let newPushKey = null
beforeAll(() => {
// Find and Delete Previous Test user, log in, get key
return User.getByUserName(testUserName)
.then((user) => {
return User.deleteUser(user?.id, password)
})
.then((results) => {
return User.register(testUserName, password)
})
.then(({ token, userId }) => {
newUserId = userId
return User.getMasterKey(userId, password)
})
.then((newMasterKey) => {
masterKey = newMasterKey
return true
})
.catch(((error) => {
console.log(error)
}))
})
test('Test Generate Push Key', () => {
return Attachment.generatePushKey(newUserId)
.then( (pushKey) => {
newPushKey = pushKey
return Attachment.generatePushKey(newUserId)
})
.then( (pushKey) => {
// expect a long, defined pushkey
expect(pushKey).toBeDefined()
expect(pushKey?.length).toBeGreaterThan(20)
expect(pushKey).toMatch(newPushKey)
})
})
test('Test get Push Key Bookmarklet', () => {
return Attachment.getPushkeyBookmarklet(newUserId)
.then(( bookmarklet => {
// Expect a bookmarklet containting URL encoded pushkey from above
const keyCheck = bookmarklet.includes(encodeURIComponent(newPushKey))
expect(bookmarklet).toBeDefined()
expect(keyCheck).toBe(true)
}))
})
test('Test Push URL', () => {
let url = 'https://www.solidscribe.com'
return Attachment.pushUrl(newPushKey, url)
.then(( results => {
return Attachment.textSearch(newUserId, 'scribe')
}))
.then((results) => {
expect(results.length == 1).toBe(true)
})
})
test('Test Delete Push Key', () => {
return Attachment.deletePushKey(newUserId)
.then(( results => {
// Expect a true bool
expect(results).toBe(true)
}))
})
afterAll(done => {
// Close Database
const db = require('../../config/database')
db.end()
done()
})

View File

@ -0,0 +1,117 @@
const Note = require('../../models/Note')
const User = require('../../models/User')
const testUserName = 'jestTestUserNote'
const password = 'Beans1234!!!'
const secondPassword = 'Rice1234!!!'
let newUserId = null
let masterKey = null
let testNoteId = 0
let testNoteId2 = 0
const searchWord1 = 'beans'
const searchWord2 = 'RICE'
const updatedNoteText = 'Some Note Text for Testing more '+searchWord2+' is nice'
beforeAll(() => {
// Find and Delete Previous Test user, log in, get key
return User.getByUserName(testUserName)
.then((user) => {
return User.deleteUser(user?.id, password)
})
.then((results) => {
return User.register(testUserName, password)
})
.then(({ token, userId }) => {
newUserId = userId
return User.getMasterKey(userId, password)
})
.then((newMasterKey) => {
masterKey = newMasterKey
return true
})
.catch(((error) => {
console.log(error)
}))
})
test('Create Note', () => {
const noteTitle = 'Test Note'
const noteText = 'Some Note Text for Testing'
return Note.create(newUserId, noteTitle, noteText, masterKey)
.then((noteId) => {
testNoteId = noteId
expect(noteId).toBeGreaterThan(0)
})
})
test('Create Another Note', () => {
const noteTitle = 'Test Note2'
const noteText = 'Some Note Text for Testing more '+searchWord1
return Note.create(newUserId, noteTitle, noteText, masterKey)
.then((noteId) => {
testNoteId2 = noteId
expect(noteId).toBeGreaterThan(0)
})
})
test('Update a note', () => {
return Note.update(newUserId, testNoteId, updatedNoteText, 'title', 0, 0, 0, 'hash', masterKey)
.then((results) => {
expect(results.changedRows).toEqual(1)
})
})
test('Decrypt a note', () => {
return Note.get(newUserId, testNoteId, masterKey)
.then((noteData) => {
expect(noteData.text).toMatch(updatedNoteText)
})
})
test('Update note search index', () => {
return Note.reindex(newUserId, masterKey)
.then((results) => {
expect(results).toBe(true)
})
})
test('Search Encrypted Index', () => {
const searchString = `${searchWord1} ${searchWord2}`
return Note.encryptedIndexSearch(newUserId, searchString, null, masterKey)
.then(({ids}) => {
// Make sure beans is in one note and rice is in updated text
expect(ids.length).toEqual(2)
})
})
test('Search Encrypted Index no results', () => {
return Note.encryptedIndexSearch(newUserId, 'zzz', null, masterKey)
.then(({ids}) => {
// Make sure beans is in one note and rice is in updated text
expect(ids.length).toEqual(0)
})
})
afterAll(done => {
// Close Database
const db = require('../../config/database')
db.end()
done()
})

View File

@ -0,0 +1,67 @@
const Note = require('../../models/Note')
const User = require('../../models/User')
const ShareNote = require('../../models/ShareNote')
const testUserName = 'jestTestUserNote'
const password = 'Beans1234!!!'
let newUserId = null
let masterKey = null
const testUserName2 = 'jestTestUserDude'
const password2 = 'Rice1234!!!'
let newUserId2 = null
let masterKey2 = null
let testNoteId = 0
let testNoteId2 = 0
// let sharedNoteId = 0 //ID of note shared with user
const shareUserId = 61
const searchWord1 = 'beans'
const searchWord2 = 'RICE'
const updatedNoteText = 'Some Note Text for Testing more '+searchWord2+' is nice'
beforeAll(() => {
// Find and Delete Previous Test user, log in, get key
return
User.getByUserName(testUserName)
.then(user => {
User.deleteUser(user?.id, password)
})
.then(user => {
User.getByUserName(testUserName2)
})
.then(user => {
User.deleteUser(user?.id, password)
})
.then((results) => {
return User.register(testUserName, password)
})
.then(({ token, userId }) => {
newUserId = userId
return User.getMasterKey(userId, password)
})
.then((newMasterKey) => {
masterKey = newMasterKey
return true
})
.catch(((error) => {
console.log(error)
}))
})
afterAll(done => {
// Close Database
const db = require('../../config/database')
db.end()
done()
})

View File

@ -0,0 +1,112 @@
const User = require('../../models/User')
const crypto = require('crypto')
const testUserName = 'jestTestUser'
const password = 'Beans1234!!!'
const secondPassword = 'Rice1234!!!'
let testUserId = null
let masterKey = null
beforeAll(() => {
// Find and Delete Previous Test user
return User.getByUserName(testUserName)
.then((user) => {
return User.deleteUser(user?.id, password)
})
.then((results) => {
return results
})
})
test('Test User Registration', () => {
return User.register(testUserName, password)
.then((({ token, userId }) => {
testUserId = userId
expect(token).toBeDefined()
expect(userId).toBeGreaterThan(0)
}))
})
test('Test decrypting user masterKey', () => {
return User.getMasterKey(testUserId, password)
.then((newMasterKey) => {
masterKey = newMasterKey
expect(masterKey).toBeDefined()
})
})
test('Test generating public and private key pair', () => {
return User.generateKeypair(testUserId, masterKey)
.then(({publicKey, privateKey}) => {
const publicKeyMessage = 'Test: Public key decrypt - Pass'
const privateKeyMessage = 'Test: Private key decrypt - Pass'
//Encrypt Message with private Key
const privateKeyEncrypted = crypto.privateEncrypt(privateKey, Buffer.from(privateKeyMessage, 'utf8')).toString('base64')
const decryptedPrivate = crypto.publicDecrypt(publicKey, Buffer.from(privateKeyEncrypted, 'base64'))
//Conver back to a string
expect(decryptedPrivate.toString('utf8')).toMatch(privateKeyMessage)
//Encrypt with public key
const pubEncrMsc = crypto.publicEncrypt(publicKey, Buffer.from(publicKeyMessage, 'utf8')).toString('base64')
const publicDeccryptMessage = crypto.privateDecrypt(privateKey, Buffer.from(pubEncrMsc, 'base64') )
//Convert it back to string
expect(publicDeccryptMessage.toString('utf8')).toMatch(publicKeyMessage)
})
})
test('Test Logging in User', () => {
return User.login(testUserName, password)
.then(({token, userId}) => {
expect(token).toBeDefined()
expect(userId).toBeGreaterThan(0)
})
})
test('Test Changing Password', () => {
return User.changePassword(testUserId, password, secondPassword)
.then((passwordChangeResults) => {
expect(passwordChangeResults).toBe(true)
})
})
test('Test Login with wrong password', () => {
return User.login(testUserName, password)
.then(({token, userId}) => {
expect(token).toBeNull()
expect(userId).toBeNull()
})
})
test('Test decrypting masterKey with new Password', () => {
return User.getMasterKey(testUserId, secondPassword)
.then((newMasterKey) => {
expect(newMasterKey).toBeDefined()
expect(newMasterKey.length).toBe(28)
})
})
afterAll(done => {
// Close Database
const db = require('../../config/database')
db.end()
done()
})

View File

@ -1,22 +0,0 @@
#!/bin/bash
# Setup env variables
source ~/.env
# Send updated dynamic IP address to Namecheap, in order to update subdomains.
# This uses curl (separate pkg) to send the change; Namecheap automatically detects source IP if the ip field (like domain, password) ..
# is not specified.
# info helper
info() { printf "\n%s %s\n\n" "$( date )" "$*" >&2; }
info "Starting IP update for subdomains"
echo "https://dynamicdns.park-your-domain.com/update?host=$DYDNS_HOST&domain=$DYDNS_DOMAIN&password=$DYDNS_PASS"
# first subdomain
curl "https://dynamicdns.park-your-domain.com/update?host=$DYDNS_HOST&domain=$DYDNS_DOMAIN&password=$DYDNS_PASS"
# second subdomain
curl "https://dynamicdns.park-your-domain.com/update?host=$DYDNS_HOST2&domain=$DYDNS_DOMAIN&password=$DYDNS_PASS"
info "IP update done"