Tons of littele interface changes and cleanups
Massive update to image scraper with much better image getter Lots of little ui updates for mobile
This commit is contained in:
@@ -6,6 +6,7 @@ const speakeasy = require('speakeasy')
|
||||
let Auth = {}
|
||||
|
||||
const tokenSecretKey = process.env.JSON_KEY
|
||||
const sessionTokenUses = 300 //Defines number of uses each session token has before being refreshed
|
||||
|
||||
//Creates session token
|
||||
Auth.createToken = (userId, masterKey, pastId = null, pastCreatedDate = null) => {
|
||||
@@ -26,7 +27,7 @@ Auth.createToken = (userId, masterKey, pastId = null, pastCreatedDate = null) =>
|
||||
|
||||
return db.promise().query(
|
||||
'INSERT INTO user_active_session (salt, encrypted_master_password, created, uses, user_hash, session_id) VALUES (?,?,?,?,?,?)',
|
||||
[salt, encryptedMasterPass, created, 40, userHash, sessionId])
|
||||
[salt, encryptedMasterPass, created, sessionTokenUses, userHash, sessionId])
|
||||
|
||||
})
|
||||
.then((r,f) => {
|
||||
|
@@ -54,7 +54,7 @@ SiteScrape.getCleanUrls = (textBlock) => {
|
||||
SiteScrape.getHostName = (url) => {
|
||||
|
||||
var hostname = 'https://'+(new URL(url)).hostname;
|
||||
console.log('hostname', hostname)
|
||||
// console.log('hostname', hostname)
|
||||
return hostname
|
||||
}
|
||||
|
||||
@@ -63,36 +63,95 @@ SiteScrape.getDisplayImage = ($, url) => {
|
||||
|
||||
const hostname = SiteScrape.getHostName(url)
|
||||
|
||||
let metaImg = $('meta[property="og:image"]')
|
||||
let shortcutIcon = $('link[rel="shortcut icon"]')
|
||||
let favicon = $('link[rel="icon"]')
|
||||
let metaImg = $('[property="og:image"]')
|
||||
let shortcutIcon = $('[rel="shortcut icon"]')
|
||||
let favicon = $('[rel="icon"]')
|
||||
let randomImg = $('img')
|
||||
|
||||
console.log('----')
|
||||
//Set of images we may want gathered from various places in source
|
||||
let imagesWeWant = []
|
||||
let thumbnail = ''
|
||||
|
||||
//Scrape metadata for page image
|
||||
//Grab the first random image we find
|
||||
if(randomImg && randomImg[0] && randomImg[0].attribs){
|
||||
thumbnail = hostname + randomImg[0].attribs.src
|
||||
console.log('random img '+thumbnail)
|
||||
if(randomImg && randomImg.length > 0){
|
||||
|
||||
let imgSrcs = []
|
||||
for (let i = 0; i < randomImg.length; i++) {
|
||||
imgSrcs.push( randomImg[i].attribs.src )
|
||||
}
|
||||
|
||||
const half = Math.ceil(imgSrcs.length / 2)
|
||||
imagesWeWant = [...imgSrcs.slice(-half), ...imgSrcs.slice(0,half) ]
|
||||
|
||||
}
|
||||
//Grab the favicon of the site
|
||||
//Grab the shortcut icon
|
||||
if(favicon && favicon[0] && favicon[0].attribs){
|
||||
thumbnail = hostname + favicon[0].attribs.href
|
||||
console.log('favicon '+thumbnail)
|
||||
imagesWeWant.push(favicon[0].attribs.href)
|
||||
}
|
||||
//Grab the shortcut icon
|
||||
if(shortcutIcon && shortcutIcon[0] && shortcutIcon[0].attribs){
|
||||
thumbnail = hostname + shortcutIcon[0].attribs.href
|
||||
console.log('shortcut '+thumbnail)
|
||||
imagesWeWant.push(shortcutIcon[0].attribs.href)
|
||||
}
|
||||
//Grab the presentation image for the site
|
||||
if(metaImg && metaImg[0] && metaImg[0].attribs){
|
||||
thumbnail = metaImg[0].attribs.content
|
||||
console.log('ogImg '+thumbnail)
|
||||
imagesWeWant.unshift(metaImg[0].attribs.content)
|
||||
}
|
||||
|
||||
// console.log(imagesWeWant)
|
||||
|
||||
//Remove everything that isn't an accepted file format
|
||||
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
|
||||
|
||||
let img = String(imagesWeWant[i])
|
||||
|
||||
if(
|
||||
!img.includes('.jpg') &&
|
||||
!img.includes('.jpeg') &&
|
||||
!img.includes('.png') &&
|
||||
!img.includes('.gif')
|
||||
){
|
||||
imagesWeWant.splice(i,1)
|
||||
}
|
||||
}
|
||||
|
||||
//Find if we have absolute thumbnails or not
|
||||
let foundAbsolute = false
|
||||
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
|
||||
|
||||
let img = imagesWeWant[i]
|
||||
|
||||
//Add host name if its not included
|
||||
if(String(img).includes('//') || String(img).includes('http')){
|
||||
foundAbsolute = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
//Go through all found images. Grab the one closest to the top. Closer is better
|
||||
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
|
||||
|
||||
let img = imagesWeWant[i]
|
||||
|
||||
if(!String(img).includes('//') && foundAbsolute){
|
||||
continue;
|
||||
}
|
||||
|
||||
//Only add host to images if no absolute images were found
|
||||
if(!String(img).includes('//') ){
|
||||
if(img.indexOf('/') != 0){
|
||||
img = '/' + img
|
||||
}
|
||||
img = hostname + img
|
||||
}
|
||||
|
||||
if(img.indexOf('//') == 0){
|
||||
img = 'https:' + img //Scrape breaks without protocol
|
||||
}
|
||||
|
||||
thumbnail = img
|
||||
|
||||
}
|
||||
|
||||
console.log('-----')
|
||||
return thumbnail
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user