I need to get back into using git. The hell is wrong with me!?
This commit is contained in:
Max G committed
1 parent
7b77bd37f3
commit
6fe39406b7
135 files changed
+53273
-699
No files matched your search
+194
-88
@@ -3,20 +3,36 @@ let db = require('@config/database')
|
||||
let Tags = require('@models/Tag')
|
||||
let Attachment = require('@models/Attachment')
|
||||
|
||||
let ProcessText = require('@helpers/ProcessText')
|
||||
|
||||
var rp = require('request-promise');
|
||||
var SolrNode = require('solr-node');
|
||||
|
||||
let Note = module.exports = {}
|
||||
|
||||
// Create client
|
||||
var client = new SolrNode({
|
||||
host: '127.0.0.1',
|
||||
port: '8983',
|
||||
core: 'note',
|
||||
protocol: 'http'
|
||||
});
|
||||
|
||||
Note.create = (userId, noteText) => {
|
||||
|
||||
Note.stressTest = () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.promise()
|
||||
.query(`
|
||||
|
||||
SELECT text FROM note;
|
||||
|
||||
`)
|
||||
.then((rows, fields) => {
|
||||
console.log()
|
||||
|
||||
rows[0].forEach(item => {
|
||||
|
||||
Note.create(68, item['text'])
|
||||
})
|
||||
|
||||
resolve(true)
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
}
|
||||
|
||||
Note.create = (userId, noteText, quickNote = 0) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
if(userId == null || userId < 10){ reject('User Id required to create note') }
|
||||
@@ -24,48 +40,95 @@ Note.create = (userId, noteText) => {
|
||||
const created = Math.round((+new Date)/1000)
|
||||
|
||||
db.promise()
|
||||
.query('INSERT INTO note (user_id, text, created) VALUES (?,?,?)', [userId, noteText, created])
|
||||
.query('INSERT INTO note (user_id, text, updated, created, quick_note) VALUES (?,?,?,?,?)',
|
||||
[userId, noteText, created, created, quickNote])
|
||||
.then((rows, fields) => {
|
||||
// New notes are empty, don't add to solr index
|
||||
// Note.reindex(userId, rows[0].insertId)
|
||||
resolve(rows[0].insertId) //Only return the new note ID when creating a new note
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
}
|
||||
|
||||
Note.update = (userId, noteId, noteText, fancyInput, color, pinned, archived) => {
|
||||
Note.reindexAll = () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
const now = Math.round((+new Date)/1000)
|
||||
|
||||
db.promise()
|
||||
.query('UPDATE note SET text = ?, raw_input = ?, pinned = ?, archived = ?, updated = ?, color = ? WHERE id = ? AND user_id = ? LIMIT 1',
|
||||
[noteText, fancyInput, pinned, archived, now, color, noteId, userId])
|
||||
.query(`
|
||||
|
||||
SELECT id, user_id FROM note;
|
||||
|
||||
`)
|
||||
.then((rows, fields) => {
|
||||
console.log()
|
||||
|
||||
rows[0].forEach(item => {
|
||||
Note.reindex(item.user_id, item.id)
|
||||
})
|
||||
|
||||
resolve(true)
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
}
|
||||
|
||||
Note.reindex = (userId, noteId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
Note.get(userId, noteId)
|
||||
.then(note => {
|
||||
const noteText = note.text
|
||||
|
||||
//Process note text and attachment data
|
||||
Attachment.scanTextForWebsites(userId, noteId, noteText).then( attachmentText => {
|
||||
Attachment.scanTextForWebsites(userId, noteId, noteText)
|
||||
.then( allNoteAttachmentText => {
|
||||
//
|
||||
// Update Solr index
|
||||
//
|
||||
Tags.string(userId, noteId).then(tagString => {
|
||||
// JSON Data
|
||||
var data = {
|
||||
'id': noteId,//string - ID of note
|
||||
'user_id': userId,//int
|
||||
'note_text': noteText,
|
||||
'note_tag': tagString,
|
||||
'attachment_text': attachmentText,
|
||||
};
|
||||
// Update document to Solr server
|
||||
client.update(data, function(err, result) {
|
||||
if (err) { console.log(err); return; }
|
||||
console.log('Note Solr Update, node/solrid ('+noteId+'):');
|
||||
console.log(result.responseHeader)
|
||||
});
|
||||
Tags.string(userId, noteId)
|
||||
.then(tagString => {
|
||||
|
||||
const fullText = ProcessText.removeHtml(noteText) +' '+ tagString +' '+ ProcessText.removeHtml(allNoteAttachmentText)
|
||||
|
||||
db.promise()
|
||||
.query(`
|
||||
|
||||
INSERT INTO note_text_index (note_id, user_id, text)
|
||||
VALUES (?,?,?)
|
||||
ON DUPLICATE KEY UPDATE text = ?
|
||||
|
||||
`, [noteId, userId, fullText, fullText])
|
||||
.then((rows, fields) => {
|
||||
resolve(true)
|
||||
})
|
||||
.catch(console.log)
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
Note.update = (userId, noteId, noteText, color, pinned, archived) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
//Prevent note loss if it saves with empty text
|
||||
if(ProcessText.removeHtml(noteText) == ''){
|
||||
console.log('Not saving empty note')
|
||||
resolve(false)
|
||||
}
|
||||
|
||||
const now = Math.round((+new Date)/1000)
|
||||
|
||||
db.promise()
|
||||
.query('UPDATE note SET text = ?, pinned = ?, archived = ?, updated = ?, color = ? WHERE id = ? AND user_id = ? LIMIT 1',
|
||||
[noteText, pinned, archived, now, color, noteId, userId])
|
||||
.then((rows, fields) => {
|
||||
|
||||
//Async solr note reindex
|
||||
Note.reindex(userId, noteId)
|
||||
|
||||
//Send back updated response
|
||||
resolve(rows[0])
|
||||
@@ -92,7 +155,25 @@ Note.delete = (userId, noteId) => {
|
||||
Note.get = (userId, noteId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.promise()
|
||||
.query('SELECT text, updated, raw_input, pinned, archived, color FROM note WHERE user_id = ? AND id = ? LIMIT 1', [userId,noteId])
|
||||
.query(`
|
||||
SELECT note.text, note.updated, note.pinned, note.archived, note.color, count(distinct attachment.id) as attachment_count
|
||||
FROM note
|
||||
LEFT JOIN attachment ON (note.id = attachment.note_id)
|
||||
WHERE note.user_id = ? AND note.id = ? LIMIT 1`, [userId,noteId])
|
||||
.then((rows, fields) => {
|
||||
|
||||
//Return note data
|
||||
resolve(rows[0][0])
|
||||
|
||||
})
|
||||
.catch(console.log)
|
||||
})
|
||||
}
|
||||
|
||||
Note.getShared = (noteId) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.promise()
|
||||
.query('SELECT text, updated, color FROM note WHERE id = ? AND shared = 1 LIMIT 1', [noteId])
|
||||
.then((rows, fields) => {
|
||||
|
||||
//Return note data
|
||||
@@ -106,56 +187,79 @@ Note.get = (userId, noteId) => {
|
||||
Note.solrQuery = (userId, searchQuery, searchTags) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
if(searchQuery != '' && searchQuery != null){
|
||||
let urlQuery = `/solr/note/select?hl.fl=note_text&hl=on&q=user_id:${userId} AND note_text:${searchQuery}&wt=json`
|
||||
urlQuery = `/solr/note/select?
|
||||
hl.fl=note_text,attachment_text,note_tag&
|
||||
hl=on&
|
||||
q=user_id:${userId} AND (note_text:${searchQuery} OR attachment_text:${searchQuery} OR note_tag:${searchQuery})&
|
||||
wt=json&
|
||||
fl=id&
|
||||
hl.fl=note_text,attachment_text,note_tag&
|
||||
hl.snippets=20&
|
||||
hl.maxAnalyzedChars=100000`
|
||||
|
||||
rp('http://127.0.0.1:8983'+urlQuery)
|
||||
.then(function (htmlString) {
|
||||
let solrResult = JSON.parse(htmlString)
|
||||
resolve(solrResult)
|
||||
})
|
||||
if(searchQuery.length == 0){
|
||||
resolve(null)
|
||||
} else {
|
||||
resolve([])
|
||||
|
||||
//Number of characters before and after search word
|
||||
const front = 5
|
||||
const tail = 150
|
||||
|
||||
db.promise()
|
||||
.query(`
|
||||
|
||||
SELECT
|
||||
note_id,
|
||||
substring(
|
||||
text,
|
||||
IF(LOCATE(?, text) > ${tail}, LOCATE(?, text) - ${front}, 1),
|
||||
${tail} + LENGTH(?) + ${front}
|
||||
) as snippet
|
||||
FROM note_text_index
|
||||
WHERE user_id = ?
|
||||
AND MATCH(text)
|
||||
AGAINST(? IN NATURAL LANGUAGE MODE)
|
||||
LIMIT 1000
|
||||
;
|
||||
|
||||
`, [searchQuery, searchQuery, searchQuery, userId, searchQuery])
|
||||
.then((rows, fields) => {
|
||||
|
||||
let results = []
|
||||
let snippets = {}
|
||||
rows[0].forEach(item => {
|
||||
let noteId = parseInt(item['note_id'])
|
||||
//Setup array of ids to use for query
|
||||
results.push( noteId )
|
||||
//Get text snippet and highlight the key word
|
||||
snippets[noteId] = item['snippet'].replace(new RegExp(searchQuery,"ig"), '<em>'+searchQuery+'</em>');
|
||||
//.replace(searchQuery,'<em>'+searchQuery+'</em>')
|
||||
})
|
||||
|
||||
resolve({ 'ids':results, 'snippets':snippets })
|
||||
})
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
Note.search = (userId, searchQuery, searchTags, fastFilters) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
//Define return data objects
|
||||
let returnData = {
|
||||
'notes':[],
|
||||
'tags':[]
|
||||
}
|
||||
|
||||
Note.solrQuery(userId, searchQuery, searchTags).then( solrResult => {
|
||||
Note.solrQuery(userId, searchQuery, searchTags).then( (textSearchResults) => {
|
||||
|
||||
let highlights = solrResult.highlighting
|
||||
let textSearchIds = []
|
||||
let highlights = {}
|
||||
|
||||
//Parse Note ID's from solr search
|
||||
let solrNoteIds = []
|
||||
if(solrResult.response){
|
||||
solrResult.response.docs.forEach(item => {
|
||||
solrNoteIds.push(parseInt(item.id))
|
||||
})
|
||||
}
|
||||
if(textSearchResults != null){
|
||||
textSearchIds = textSearchResults['ids']
|
||||
highlights = textSearchResults['snippets']
|
||||
}
|
||||
|
||||
|
||||
|
||||
//No results, return empty data
|
||||
if(solrNoteIds.length == 0 && searchQuery.length > 0){
|
||||
resolve(returnData)
|
||||
if(textSearchIds.length == 0 && searchQuery.length > 0){
|
||||
return resolve(returnData)
|
||||
}
|
||||
|
||||
//Default note lookup gets all notes
|
||||
// Add to query for character counts -> CHAR_LENGTH(note.text) as chars
|
||||
let noteSearchQuery = `
|
||||
SELECT note.id,
|
||||
SUBSTRING(note.text, 1, 400) as text,
|
||||
@@ -166,20 +270,15 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
|
||||
note.archived
|
||||
FROM note
|
||||
LEFT JOIN note_tag ON (note.id = note_tag.note_id)
|
||||
LEFT JOIN attachment ON (note.id = attachment.note_id AND attachment.attachment_type = 1)
|
||||
LEFT JOIN attachment ON (note.id = attachment.note_id)
|
||||
WHERE note.user_id = ?`
|
||||
let searchParams = [userId]
|
||||
|
||||
if(solrNoteIds.length > 0){
|
||||
searchParams.push(solrNoteIds)
|
||||
if(textSearchIds.length > 0){
|
||||
searchParams.push(textSearchIds)
|
||||
noteSearchQuery += ' AND note.id IN (?)'
|
||||
}
|
||||
|
||||
// if(searchQuery != ''){
|
||||
// //If a search query is defined, search notes for that word
|
||||
// searchParams.push('%'+searchQuery+'%')
|
||||
// noteSearchQuery += ' AND text LIKE ?'
|
||||
// }
|
||||
if(searchTags.length > 0){
|
||||
//If tags are passed, use those tags in search
|
||||
searchParams.push(searchTags)
|
||||
@@ -211,6 +310,9 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
|
||||
noteSearchQuery += ' HAVING note.archived = 1'
|
||||
}
|
||||
|
||||
//
|
||||
// Always prioritize pinned notes in searches.
|
||||
|
||||
//Default Sort, order by last updated
|
||||
let defaultOrderBy = ' ORDER BY note.pinned DESC, updated DESC, created DESC, opened DESC, id DESC'
|
||||
|
||||
@@ -226,6 +328,19 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
|
||||
//Append Order by to query
|
||||
noteSearchQuery += defaultOrderBy
|
||||
|
||||
|
||||
//Manage limit params if set
|
||||
if(fastFilters.limitSize > 0 || fastFilters.limitOffset > 0){
|
||||
|
||||
const limitSize = parseInt(fastFilters.limitSize, 10) || 10 //Use int or default to 10
|
||||
const limitOffset = parseInt(fastFilters.limitOffset, 10) || 0 //Either parse int, or use zero
|
||||
|
||||
|
||||
console.log(` LIMIT ${limitOffset}, ${limitSize}`)
|
||||
noteSearchQuery += ` LIMIT ${limitOffset}, ${limitSize}`
|
||||
}
|
||||
|
||||
|
||||
db.promise()
|
||||
.query(noteSearchQuery, searchParams)
|
||||
.then((noteRows, noteFields) => {
|
||||
@@ -239,6 +354,8 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
|
||||
|
||||
//Grab note ID for finding tags
|
||||
noteIds.push(note.id)
|
||||
|
||||
if(note.text == null){ note.text = '' }
|
||||
|
||||
//Attempt to pull string out of first tag in note
|
||||
let reg = note.text.match(/<([\w]+)[^>]*>(.*?)<\/\1>/g)
|
||||
@@ -251,18 +368,13 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
|
||||
}
|
||||
|
||||
//Clean up html title
|
||||
note.title = note.title
|
||||
.replace(/&[#A-Za-z0-9]+;/g,'') //Rip out all HTML entities
|
||||
.replace(/<[^>]+>/g, '') //Rip out all HTML tags
|
||||
note.title = ProcessText.removeHtml(note.title)
|
||||
|
||||
//Generate Subtext
|
||||
note.subtext = ''
|
||||
if(note.text != '' && note.title != ''){
|
||||
note.subtext = note.text
|
||||
.replace(/&[#A-Za-z0-9]+;/g,' ') //Rip out all HTML entities
|
||||
.replace(/<[^>]+>/g, ' ') //Rip out all HTML tags
|
||||
.replace(/\s+/g, ' ') //Remove all whitespace
|
||||
.substring(note.title.length + 2)
|
||||
note.subtext = ProcessText.removeHtml(note.text)
|
||||
.substring(note.title.length)
|
||||
}
|
||||
|
||||
|
||||
@@ -271,14 +383,8 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
|
||||
note.tag_highlights = []
|
||||
|
||||
//Push in solr highlights
|
||||
if(highlights && highlights[note.id] && highlights[note.id].note_text){
|
||||
note['note_highlights'] = highlights[note.id].note_text
|
||||
}
|
||||
if(highlights && highlights[note.id] && highlights[note.id].attachment_text){
|
||||
note['attachment_highlights'] = highlights[note.id].attachment_text
|
||||
}
|
||||
if(highlights && highlights[note.id] && highlights[note.id].note_tag){
|
||||
note['tag_highlights'] = highlights[note.id].note_tag
|
||||
if(highlights && highlights[note.id]){
|
||||
note['note_highlights'] = [highlights[note.id]]
|
||||
}
|
||||
|
||||
//Clear out note.text before sending it to front end
|
||||
@@ -287,7 +393,7 @@ Note.search = (userId, searchQuery, searchTags, fastFilters) => {
|
||||
|
||||
//If no notes are returned, there are no tags, return empty
|
||||
if(noteIds.length == 0){
|
||||
resolve(returnData)
|
||||
return resolve(returnData)
|
||||
}
|
||||
|
||||
//Only show tags of selected notes
|
||||
|
||||
Reference in new issue
Block a user