Max/SolidScribe
1
0 Fork 0
Code Issues 24 Pull Requests Releases Wiki Activity

Scraper Update for testing

Browse Source
This commit is contained in:
Max committed 2026-11-03 10:23:03 -07:00
1 parent 7f38444707
commit 26e31f1885
8 files changed
+774 -750
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

jest.config.js
+15 -7
View File
@@ -1,11 +1,19 @@
const path = '../../'
const prefix = '/$1'
//
// The @alias map must mirror _moduleAliases in package.json, which is what
// module-alias/register sets up at runtime.
//
// These were previously written as "../../server/..." relative paths, which
// jest resolves relative to the requiring file - so they never actually
// resolved, and every existing test sidestepped them with relative requires.
// <rootDir> is depth-independent and works from any test directory.
//
module.exports = {
moduleNameMapper: {
"@root/(.*)": ".",
"@models/(.*)": path+"server/models"+prefix,
"@routes/(.*)": path+"server/routes"+prefix,
"@helpers/(.*)": path+"server/helpers"+prefix,
"@config/(.*)": path+"server/config"+prefix,
"^@root/(.*)$": "<rootDir>/$1",
"^@models/(.*)$": "<rootDir>/server/models/$1",
"^@routes/(.*)$": "<rootDir>/server/routes/$1",
"^@helpers/(.*)$": "<rootDir>/server/helpers/$1",
"^@config/(.*)$": "<rootDir>/server/config/$1",
"^@workers/(.*)$": "<rootDir>/server/workers/$1",
}
}
package.json
+3 -4
View File
@@ -10,7 +10,7 @@
"license": "ISC",
"dependencies": {
"body-parser": "^1.19.0",
"cheerio": "^1.0.0-rc.3",
"cheerio": "^1.1.0",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"express-rate-limit": "^5.1.3",
@@ -22,8 +22,6 @@
"mysql2": "^2.2.5",
"node-tesseract-ocr": "^2.0.0",
"qrcode": "^1.4.4",
"request": "^2.88.2",
"request-promise": "^4.2.6",
"socket.io": "^2.3.0",
"speakeasy": "^2.0.0"
},
@@ -32,7 +30,8 @@
"@models": "server/models",
"@routes": "server/routes",
"@helpers": "server/helpers",
"@config": "server/config"
"@config": "server/config",
"@workers": "server/workers"
},
"devDependencies": {
"jest": "^29.7.0"
pnpm-lock.yaml
Generated
+1 -358
View File
@@ -12,7 +12,7 @@ importers:
specifier: ^1.19.0
version: 1.20.9([email protected])
cheerio:
specifier: ^1.0.0-rc.3
specifier: ^1.1.0
version: 1.2.0
dotenv:
specifier: ^8.2.0
@@ -47,12 +47,6 @@ importers:
qrcode:
specifier: ^1.4.4
version: 1.5.4
request:
specifier: ^2.88.2
version: 2.88.2
request-promise:
specifier: ^4.2.6
version: 4.2.6([email protected])
socket.io:
specifier: ^2.3.0
version: 2.5.1([email protected])
@@ -373,9 +367,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-QbJ0NTQ/I9DI3uSJA4cbexiwQeRAfjPScqIbSjUDd9TOrcg6pTkdgziesOqxBMBzit8vFCTwrP27t13vFOORRA==}
[email protected]:
resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==}
[email protected]:
resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==}
engines: {node: '>=8'}
@@ -414,22 +405,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==}
[email protected]:
resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==}
[email protected]:
resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==}
engines: {node: '>=0.8'}
[email protected]:
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
[email protected]:
resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==}
[email protected]:
resolution: {integrity: sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==}
[email protected]:
resolution: {integrity: sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@@ -477,15 +452,9 @@ packages:
engines: {node: '>=6.0.0'}
hasBin: true
[email protected]:
resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==}
[email protected]:
resolution: {integrity: sha512-gaqbzQPqOoamawKg0LGVd7SzLgXS+JH61oWprSLH+P+abTczqJbhTR8CmJ2u9/bUYNmHTGJx/UEmn6doAvvuig==}
[email protected]:
resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
[email protected]:
resolution: {integrity: sha512-FISIQxV5u91Y9gvuFq3q/tA/qTqLJn3AhRcEvyozyhJTh5OBFFtv827AdHU7Rbyct4r+Ba1/99oFa1Y3yuFsdw==}
engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@@ -545,9 +514,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-LFUqIIAwpzsIuevnKnsR9qnStBcgzPShTX61lBwN9uFkkf8o8zLiebEjbW8AYtZiF+1YKDWfdt/pWDtMuG8Xwg==}
[email protected]:
resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==}
[email protected]:
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
engines: {node: '>=10'}
@@ -591,10 +557,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
[email protected]:
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
engines: {node: '>= 0.8'}
[email protected]:
resolution: {integrity: sha512-WZveuKPeKAG9qY+FkYDeADzdHyTYdIboXS59ixDeRJL5ZhxpqUnxSOwop4FQjMsiYm3/Or8cegVbpAHNA7pHxw==}
@@ -636,9 +598,6 @@ packages:
resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==}
engines: {node: '>= 0.6'}
[email protected]:
resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==}
[email protected]:
resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
@@ -658,10 +617,6 @@ packages:
resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==}
engines: {node: '>= 6'}
[email protected]:
resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==}
engines: {node: '>=0.10'}
[email protected]:
resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
peerDependencies:
@@ -720,10 +675,6 @@ packages:
resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==}
engines: {node: '>=0.10.0'}
[email protected]:
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
[email protected]:
resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
engines: {node: '>=0.10'}
@@ -772,9 +723,6 @@ packages:
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
[email protected]:
resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==}
[email protected]:
resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==}
@@ -874,16 +822,6 @@ packages:
resolution: {integrity: sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw==}
engines: {node: '>= 0.10.0'}
[email protected]:
resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
[email protected]:
resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==}
engines: {'0': node >=0.6.0}
[email protected]:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
[email protected]:
resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}
@@ -902,13 +840,6 @@ packages:
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
engines: {node: '>=8'}
[email protected]:
resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==}
[email protected]:
resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==}
engines: {node: '>= 0.12'}
[email protected]:
resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
engines: {node: '>= 0.6'}
@@ -955,9 +886,6 @@ packages:
resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==}
engines: {node: '>=10'}
[email protected]:
resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==}
[email protected]:
resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]
@@ -974,15 +902,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
[email protected]:
resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==}
engines: {node: '>=4'}
[email protected]:
resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==}
engines: {node: '>=6'}
deprecated: this library is no longer supported
[email protected]:
resolution: {integrity: sha512-G1LWKhDSvhGeAQ8mPVQlqNcOB2sJdwATtZKl2pDKKHfpf/rYj24lkinxf69blJbnsvtqqNU+L3SL50vzZhXOnw==}
@@ -1015,10 +934,6 @@ packages:
resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==}
engines: {node: '>= 0.8'}
[email protected]:
resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==}
engines: {node: '>=0.8', npm: '>=1.3.7'}
[email protected]:
resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==}
engines: {node: '>=10.17.0'}
@@ -1080,9 +995,6 @@ packages:
resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
engines: {node: '>=8'}
[email protected]:
resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==}
[email protected]:
resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==}
@@ -1095,9 +1007,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
[email protected]:
resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==}
[email protected]:
resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==}
engines: {node: '>=8'}
@@ -1258,9 +1167,6 @@ packages:
resolution: {integrity: sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==}
hasBin: true
[email protected]:
resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==}
[email protected]:
resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==}
engines: {node: '>=6'}
@@ -1269,15 +1175,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
[email protected]:
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
[email protected]:
resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==}
[email protected]:
resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
[email protected]:
resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
engines: {node: '>=6'}
@@ -1287,10 +1184,6 @@ packages:
resolution: {integrity: sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==}
engines: {node: '>=4', npm: '>=1.4.28'}
[email protected]:
resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==}
engines: {node: '>=0.6.0'}
[email protected]:
resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==}
@@ -1333,9 +1226,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==}
[email protected]:
resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==}
[email protected]:
resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==}
@@ -1457,9 +1347,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
[email protected]:
resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==}
[email protected]:
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
engines: {node: '>=0.10.0'}
@@ -1536,9 +1423,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==}
[email protected]:
resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==}
[email protected]:
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
@@ -1573,13 +1457,6 @@ packages:
resolution: {integrity: sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==}
engines: {node: '>= 0.10'}
[email protected]:
resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==}
[email protected]:
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
[email protected]:
resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==}
@@ -1592,10 +1469,6 @@ packages:
resolution: {integrity: sha512-5Px6J7rfHBAGUhNQ3es4YzcOdBBw0SDcVtThJml4RdnKw4tKs2epA0kPZ+K8X+WiHktT93RHVcQ/3ym5aTCgtg==}
engines: {node: '>=0.6'}
[email protected]:
resolution: {integrity: sha512-mzR4sElr1bfCaPJe7m8ilJ6ZXdDaGoObcYR0ZHSsktM/Lt21MVHj5De30GQH2eiZ1qGRTO7LCAzQsUeXTNexWQ==}
engines: {node: '>=0.6'}
[email protected]:
resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
engines: {node: '>= 0.6'}
@@ -1613,24 +1486,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
[email protected]:
resolution: {integrity: sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==}
engines: {node: '>=0.10.0'}
peerDependencies:
request: ^2.34
[email protected]:
resolution: {integrity: sha512-HCHI3DJJUakkOr8fNoCc73E5nU5bqITjOYFMDrKHYOXWXrgD/SBaC7LjwuPymUprRyuF06UK7hd/lMHkmUXglQ==}
engines: {node: '>=0.10.0'}
deprecated: request-promise has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142
peerDependencies:
request: ^2.34
[email protected]:
resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==}
engines: {node: '>= 6'}
deprecated: request has been deprecated, see https://github.com/request/request/issues/3142
[email protected]:
resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
engines: {node: '>=0.10.0'}
@@ -1762,11 +1617,6 @@ packages:
resolution: {integrity: sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==}
engines: {node: '>= 0.6'}
[email protected]:
resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==}
engines: {node: '>=0.10.0'}
hasBin: true
[email protected]:
resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==}
engines: {node: '>=10'}
@@ -1775,10 +1625,6 @@ packages:
resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==}
engines: {node: '>= 0.8'}
[email protected]:
resolution: {integrity: sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g==}
engines: {node: '>=0.10.0'}
[email protected]:
resolution: {integrity: sha512-jos8u++JKm0ARcSUTAZXOVC0mSox7Bhn6sBgty73P1f3JGf7yG2clTbBNHUdde/kdvP2FESam+vM6l8jBrNxHA==}
engines: {node: '>=0.8.0'}
@@ -1843,16 +1689,6 @@ packages:
resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
engines: {node: '>=0.6'}
[email protected]:
resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==}
engines: {node: '>=0.8'}
[email protected]:
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
[email protected]:
resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==}
[email protected]:
resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==}
engines: {node: '>=4'}
@@ -1885,9 +1721,6 @@ packages:
peerDependencies:
browserslist: '>= 4.21.0'
[email protected]:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
[email protected]:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
@@ -1895,11 +1728,6 @@ packages:
resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}
engines: {node: '>= 0.4.0'}
[email protected]:
resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==}
deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).
hasBin: true
[email protected]:
resolution: {integrity: sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==}
engines: {node: '>=10.12.0'}
@@ -1908,10 +1736,6 @@ packages:
resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}
engines: {node: '>= 0.8'}
[email protected]:
resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==}
engines: {'0': node >=0.6.0}
[email protected]:
resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==}
@@ -2449,13 +2273,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
fast-deep-equal: 3.1.3
fast-json-stable-stringify: 2.1.0
json-schema-traverse: 0.4.1
uri-js: 4.4.1
[email protected]:
dependencies:
type-fest: 0.21.3
@@ -2487,18 +2304,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
safer-buffer: 2.1.2
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected](@babel/[email protected]([email protected]))([email protected]):
dependencies:
'@babel/core': 7.29.7([email protected])
@@ -2566,14 +2371,8 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
tweetnacl: 0.14.5
[email protected]: {}
[email protected]: {}
[email protected]([email protected]):
dependencies:
bytes: 3.1.2
@@ -2643,8 +2442,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]:
dependencies:
ansi-styles: 4.3.0
@@ -2701,10 +2498,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
delayed-stream: 1.0.0
[email protected]: {}
[email protected]: {}
@@ -2736,8 +2529,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected](@types/[email protected])([email protected]):
@@ -2771,10 +2562,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
assert-plus: 1.0.0
[email protected]([email protected]):
dependencies:
ms: 2.0.0
@@ -2811,8 +2598,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -2856,11 +2641,6 @@ snapshots:
es-errors: 1.3.0
gopd: 1.2.0
[email protected]:
dependencies:
jsbn: 0.1.1
safer-buffer: 2.1.2
[email protected]:
dependencies:
safe-buffer: 5.2.1
@@ -3007,12 +2787,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3040,14 +2814,6 @@ snapshots:
locate-path: 5.0.0
path-exists: 4.0.0
[email protected]: {}
[email protected]:
dependencies:
asynckit: 0.4.0
combined-stream: 1.0.8
mime-types: 2.1.35
[email protected]: {}
[email protected]: {}
@@ -3089,10 +2855,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
assert-plus: 1.0.0
[email protected]:
dependencies:
fs.realpath: 1.0.0
@@ -3115,13 +2877,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]:
dependencies:
ajv: 6.15.0
har-schema: 2.0.0
[email protected]:
dependencies:
isarray: 2.0.1
@@ -3155,12 +2910,6 @@ snapshots:
statuses: 2.0.2
toidentifier: 1.0.1
[email protected]:
dependencies:
assert-plus: 1.0.0
jsprim: 1.4.2
sshpk: 1.18.0
[email protected]: {}
[email protected]:
@@ -3205,8 +2954,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -3215,8 +2962,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]([email protected]):
@@ -3573,18 +3318,10 @@ snapshots:
argparse: 1.0.10
esprima: 4.0.1
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3600,13 +3337,6 @@ snapshots:
ms: 2.1.3
semver: 5.7.2
[email protected]:
dependencies:
assert-plus: 1.0.0
extsprintf: 1.3.0
json-schema: 0.4.0
verror: 1.10.0
[email protected]:
dependencies:
buffer-equal-constant-time: 1.0.1
@@ -3642,8 +3372,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3751,8 +3479,6 @@ snapshots:
dependencies:
boolbase: 1.0.0
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -3819,8 +3545,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -3851,12 +3575,6 @@ snapshots:
forwarded: 0.2.0
ipaddr.js: 1.9.1
[email protected]:
dependencies:
punycode: 2.3.1
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3870,8 +3588,6 @@ snapshots:
es-define-property: 1.0.1
side-channel: 1.1.1
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3900,42 +3616,6 @@ snapshots:
string_decoder: 1.1.1
util-deprecate: 1.0.2
[email protected]([email protected]):
dependencies:
lodash: 4.18.1
request: 2.88.2
[email protected]([email protected]):
dependencies:
bluebird: 3.7.2
request: 2.88.2
request-promise-core: 1.1.4([email protected])
stealthy-require: 1.1.1
tough-cookie: 2.5.0
[email protected]:
dependencies:
aws-sign2: 0.7.0
aws4: 1.13.2
caseless: 0.12.0
combined-stream: 1.0.8
extend: 3.0.2
forever-agent: 0.6.1
form-data: 2.3.3
har-validator: 5.1.5
http-signature: 1.2.0
is-typedarray: 1.0.0
isstream: 0.1.2
json-stringify-safe: 5.0.1
mime-types: 2.1.35
oauth-sign: 0.9.0
performance-now: 2.1.0
qs: 6.5.5
safe-buffer: 5.2.1
tough-cookie: 2.5.0
tunnel-agent: 0.6.0
uuid: 3.4.0
[email protected]: {}
[email protected]: {}
@@ -4104,26 +3784,12 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
asn1: 0.2.6
assert-plus: 1.0.0
bcrypt-pbkdf: 1.0.2
dashdash: 1.14.1
ecc-jsbn: 0.1.2
getpass: 0.1.7
jsbn: 0.1.1
safer-buffer: 2.1.2
tweetnacl: 0.14.5
[email protected]:
dependencies:
escape-string-regexp: 2.0.0
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -4179,17 +3845,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
psl: 1.15.0
punycode: 2.3.1
[email protected]:
dependencies:
safe-buffer: 5.2.1
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -4213,16 +3868,10 @@ snapshots:
escalade: 3.2.0
picocolors: 1.1.1
[email protected]:
dependencies:
punycode: 2.3.1
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
dependencies:
'@jridgewell/trace-mapping': 0.3.31
@@ -4231,12 +3880,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
assert-plus: 1.0.0
core-util-is: 1.0.2
extsprintf: 1.3.0
[email protected]:
dependencies:
makeerror: 1.0.12
server/helpers/SiteScrape.js
+361 -134
View File
@@ -1,8 +1,15 @@
let SiteScrape = module.exports = {}
//
// Pure parsing of an already-fetched page. No network I/O, no logging.
//
// $ = the cheerio scrape object
//
// Everything here is a pure function of ($, url), which is what makes it
// testable against saved HTML fixtures instead of the live internet.
//
const config = require('@config/scrape')
let SiteScrape = module.exports = {}
const removeWhitespace = /\s+/g
@@ -13,162 +20,311 @@ const commonWords = ['just','start','what','these','how', 'was', 'being','can','
'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
SiteScrape.getTitle = ($) => {
let title = $('title').text().replace(removeWhitespace, " ")
return title
const clean = (value, maxLength) => {
if (value == null) { return '' }
const text = String(value).replace(removeWhitespace, ' ').trim()
return maxLength ? text.substring(0, maxLength) : text
}
//Finds all urls in text, removes duplicates, makes sure they have https://
SiteScrape.getCleanUrls = (textBlock) => {
//Find all URLs in text
//@TODO - Use the process text library for this function
const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
let allUrls = textBlock.match(urlPattern)
if(allUrls == null){
return []
// First non-empty wins. Each candidate is a thunk so a missing branch costs
// nothing and nothing throws on the way to the answer.
const firstOf = (candidates) => {
for (const candidate of candidates) {
let value = ''
try { value = clean(typeof candidate === 'function' ? candidate() : candidate) }
catch (error) { value = '' }
if (value) { return value }
}
return ''
}
//Every URL needs HTTPS!!!
let foundUrls = []
allUrls.forEach( (item, index) => {
//add protocol if it is missing
if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
allUrls[index] = 'https://'+item
const metaContent = ($, selector) => {
const node = $(selector).first()
return node.length ? node.attr('content') : ''
}
//convert http to https
if(item.indexOf('http://') >= 0){
allUrls[index] = item.replace('http://','https://')
//
// JSON-LD. Malformed JSON-LD is extremely common, so every parse is guarded
// and a bad block is skipped rather than failing the whole page.
//
const LD_TYPES = ['Article', 'NewsArticle', 'BlogPosting', 'WebPage', 'Product', 'VideoObject', 'Recipe']
SiteScrape.getJsonLd = ($) => {
const blocks = []
$('script[type="application/ld+json"]').each((index, element) => {
const raw = $(element).contents().text()
if (!raw || !raw.trim()) { return }
let parsed = null
try { parsed = JSON.parse(raw) } catch (error) { return }
// A block may be an object, an array, or wrap its payload in @graph.
const flatten = (node) => {
if (!node || typeof node !== 'object') { return }
if (Array.isArray(node)) { return node.forEach(flatten) }
if (Array.isArray(node['@graph'])) { node['@graph'].forEach(flatten) }
blocks.push(node)
}
flatten(parsed)
})
//Remove all duplicates
foundUrls = [...new Set(allUrls)]
if (!blocks.length) { return null }
return foundUrls
// Prefer a block whose @type looks like page content.
const typed = blocks.find(block => {
const type = block['@type']
const types = Array.isArray(type) ? type : [type]
return types.some(entry => LD_TYPES.includes(entry))
})
return typed || blocks[0]
}
const jsonLdImage = (block) => {
if (!block || !block.image) { return '' }
const image = Array.isArray(block.image) ? block.image[0] : block.image
if (!image) { return '' }
if (typeof image === 'string') { return image }
if (typeof image === 'object') { return image.url || image.contentUrl || '' }
return ''
}
//
// Page title.
//
SiteScrape.getTitle = ($) => {
const ld = SiteScrape.getJsonLd($)
return firstOf([
() => metaContent($, 'meta[property="og:title"]'),
() => metaContent($, 'meta[name="twitter:title"]'),
() => ld && (ld.headline || ld.name),
() => $('title').first().text(),
() => $('h1').first().text(),
]).substring(0, 500)
}
//
// Page description. The old parser extracted none at all.
//
SiteScrape.getDescription = ($) => {
const ld = SiteScrape.getJsonLd($)
return firstOf([
() => metaContent($, 'meta[property="og:description"]'),
() => metaContent($, 'meta[name="twitter:description"]'),
() => metaContent($, 'meta[name="description"]'),
() => ld && ld.description,
]).substring(0, 1000)
}
SiteScrape.getSiteName = ($, url) => {
return firstOf([
() => metaContent($, 'meta[property="og:site_name"]'),
() => metaContent($, 'meta[name="application-name"]'),
() => {
try { return new URL(url).hostname.replace(/^www\./, '') }
catch (error) { return '' }
},
]).substring(0, 255)
}
//
// Canonical URL. Used as the base for resolving relative asset paths, which is
// what the old manual hostname-concatenation got wrong.
//
SiteScrape.getCanonicalUrl = ($, base) => {
const candidate = firstOf([
() => $('link[rel="canonical"]').first().attr('href'),
() => metaContent($, 'meta[property="og:url"]'),
])
if (!candidate) { return base || '' }
try { return new URL(candidate, base || undefined).href }
catch (error) { return base || '' }
}
//Site hostname with https:// eg: https://www.google.com
SiteScrape.getHostName = (url) => {
try {
return 'https://' + (new URL(url)).hostname
} catch (error) {
return ''
}
}
var hostname = 'https://'+(new URL(url)).hostname;
// console.log('hostname', hostname)
return hostname
//
// Image selection.
//
// Extension allowlist plus an allow-if-extensionless rule. The old substring
// filter required .jpg/.jpeg/.png/.gif anywhere in the string, which silently
// dropped .webp, .avif, .ico and - more importantly - every extensionless
// og:image endpoint like /og?title=x, which is what most modern sites serve.
//
const IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.avif', '.ico', '.bmp', '.jfif']
const TRACKING_PIXEL = /(^|[\/_\-.])(1x1|pixel|spacer|blank|transparent|dot)\.(gif|png)/i
const acceptableImagePath = (href) => {
let pathname = ''
try { pathname = new URL(href).pathname.toLowerCase() }
catch (error) { return false }
const lastSegment = pathname.split('/').pop() || ''
const dot = lastSegment.lastIndexOf('.')
// No extension at all - allow it. An og:image endpoint is the common case.
if (dot === -1) { return true }
const extension = lastSegment.slice(dot)
if (extension === '.svg') { return !!config.allowSvgThumbnails }
return IMAGE_EXTENSIONS.includes(extension)
}
// Resolve against the page's canonical base. Handles protocol-relative
// (//host/x), root-relative, and path-relative uniformly - no string surgery.
const resolveImage = (src, base) => {
if (!src) { return '' }
const value = String(src).trim()
if (!value || value.startsWith('data:')) { return '' }
try { return new URL(value, base).href }
catch (error) { return '' }
}
// Largest candidate in a srcset, by width descriptor.
const largestFromSrcset = (srcset) => {
if (!srcset) { return '' }
let best = ''
let bestWidth = -1
for (const entry of String(srcset).split(',')) {
const parts = entry.trim().split(/\s+/)
if (!parts[0]) { continue }
const descriptor = parts[1] || ''
const width = descriptor.endsWith('w') ? parseInt(descriptor, 10) : 0
if (width > bestWidth) {
bestWidth = width
best = parts[0]
}
}
return best
}
const iconCandidates = ($) => {
const icons = []
// apple-touch-icon is usually the largest and cleanest of the icons.
$('link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"]').each((index, element) => {
const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
icons.push({ href: $(element).attr('href'), weight: 1000 + sizes })
})
$('link[rel="icon"], link[rel="shortcut icon"]').each((index, element) => {
const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
icons.push({ href: $(element).attr('href'), weight: sizes })
})
return icons
.sort((a, b) => b.weight - a.weight)
.map(icon => icon.href)
}
const inlineImageCandidates = ($) => {
const images = []
$('img').each((index, element) => {
const node = $(element)
// Guarded: the old code read element.attribs.src directly and threw on
// an <img> with no attributes.
const src = node.attr('src') || node.attr('data-src') || largestFromSrcset(node.attr('srcset'))
if (!src) { return }
const width = parseInt(node.attr('width') || '0', 10) || 0
const height = parseInt(node.attr('height') || '0', 10) || 0
// Tracking pixels and spacers.
if ((width > 0 && width <= 1) || (height > 0 && height <= 1)) { return }
if (TRACKING_PIXEL.test(src)) { return }
images.push({ src, area: width * height, order: index })
})
// Biggest first; ties keep document order, so the top of the page wins.
return images
.sort((a, b) => (b.area - a.area) || (a.order - b.order))
.map(image => image.src)
}
//
// Ordered candidate list, so "which image" is an explicit priority rather than
// the old reverse-iterate-and-overwrite loop that only worked by accident.
//
SiteScrape.getImageCandidates = ($, url) => {
const base = SiteScrape.getCanonicalUrl($, url) || url
const ld = SiteScrape.getJsonLd($)
const raw = [
metaContent($, 'meta[property="og:image:secure_url"]'),
metaContent($, 'meta[property="og:image:url"]'),
metaContent($, 'meta[property="og:image"]'),
metaContent($, 'meta[name="twitter:image"]'),
metaContent($, 'meta[name="twitter:image:src"]'),
jsonLdImage(ld),
...inlineImageCandidates($),
...iconCandidates($),
]
const seen = new Set()
const candidates = []
for (const entry of raw) {
const resolved = resolveImage(entry, base)
if (!resolved || seen.has(resolved)) { continue }
if (!acceptableImagePath(resolved)) { continue }
seen.add(resolved)
candidates.push(resolved)
}
return candidates
}
// URL for image that can be downloaded to represent website
SiteScrape.getDisplayImage = ($, url) => {
const hostname = SiteScrape.getHostName(url)
let metaImg = $('[property="og:image"]')
let shortcutIcon = $('[rel="shortcut icon"]')
let favicon = $('[rel="icon"]')
let randomImg = $('img')
//Set of images we may want gathered from various places in source
let imagesWeWant = []
let thumbnail = ''
//Scrape metadata for page image
if(randomImg && randomImg.length > 0){
let imgSrcs = []
for (let i = 0; i < randomImg.length; i++) {
imgSrcs.push( randomImg[i].attribs.src )
}
const half = Math.ceil(imgSrcs.length / 2)
imagesWeWant = [...imgSrcs.slice(-half), ...imgSrcs.slice(0,half) ]
}
//Grab the shortcut icon
if(favicon && favicon[0] && favicon[0].attribs){
imagesWeWant.push(favicon[0].attribs.href)
}
//Grab the shortcut icon
if(shortcutIcon && shortcutIcon[0] && shortcutIcon[0].attribs){
imagesWeWant.push(shortcutIcon[0].attribs.href)
}
//Grab the presentation image for the site
if(metaImg && metaImg[0] && metaImg[0].attribs){
imagesWeWant.unshift(metaImg[0].attribs.content)
}
// console.log(imagesWeWant)
//Remove everything that isn't an accepted file format
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
let img = String(imagesWeWant[i])
if(
!img.includes('.jpg') &&
!img.includes('.jpeg') &&
!img.includes('.png') &&
!img.includes('.gif')
){
imagesWeWant.splice(i,1)
}
}
//Find if we have absolute thumbnails or not
let foundAbsolute = false
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
let img = imagesWeWant[i]
//Add host name if its not included
if(String(img).includes('//') || String(img).includes('http')){
foundAbsolute = true
break
}
}
//Go through all found images. Grab the one closest to the top. Closer is better
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
let img = imagesWeWant[i]
if(!String(img).includes('//') && foundAbsolute){
continue;
}
//Only add host to images if no absolute images were found
if(!String(img).includes('//') ){
if(img.indexOf('/') != 0){
img = '/' + img
}
img = hostname + img
}
if(img.indexOf('//') == 0){
img = 'https:' + img //Scrape breaks without protocol
}
thumbnail = img
}
return thumbnail
const candidates = SiteScrape.getImageCandidates($, url)
return candidates.length ? candidates[0] : ''
}
// Get all the site text and parse out the words that appear most
SiteScrape.getKeywords = ($) => {
let majorContent = ''
// Strip chrome and scripts before counting, otherwise navigation labels and
// inline JS identifiers dominate the frequency table.
const working = $.root().clone()
working.find('script, style, noscript, nav, header, footer, aside, form, svg').remove()
majorContent += $('[class*=content]').text()
.replace(removeWhitespace, " ") //Remove all whitespace
// .replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,6000) //Limit to 6000 characters
let source = working.find('article, main, [role=main], [class*=content]').text()
if (!source || source.trim().length < 200) {
source = working.find('body').text() || working.text()
}
const majorContent = String(source)
.replace(removeWhitespace, ' ')
.substring(0, config.maxKeywordSourceChars)
.toLowerCase()
.replace(/[^A-Za-z0-9- ]/g, '');
console.log(majorContent)
.replace(/[^A-Za-z0-9- ]/g, '')
//Count frequency of each word in scraped text
let frequency = {}
@@ -213,7 +369,78 @@ SiteScrape.getKeywords = ($) => {
return ''
}
SiteScrape.getMainText = ($) => {}
//
// The plaintext blob stored in attachment.text and indexed by the MySQL
// FULLTEXT index. Must stay unencrypted for MATCH ... AGAINST to work.
//
SiteScrape.buildSearchText = ({ title, description, siteName, keywords }) => {
const parts = []
if (title) { parts.push(title) }
if (description) { parts.push(description) }
if (siteName) { parts.push(siteName) }
if (keywords) { parts.push(keywords) }
return parts.join('\n').substring(0, config.maxSearchTextBytes)
}
//
// One call, everything a link preview needs.
//
SiteScrape.parse = ($, url) => {
const canonicalUrl = SiteScrape.getCanonicalUrl($, url)
const title = SiteScrape.getTitle($)
const description = SiteScrape.getDescription($)
const siteName = SiteScrape.getSiteName($, url)
const keywords = SiteScrape.getKeywords($)
const imageCandidates = SiteScrape.getImageCandidates($, url)
return {
title,
description,
siteName,
canonicalUrl,
keywords,
imageUrl: imageCandidates.length ? imageCandidates[0] : '',
imageCandidates,
hostname: SiteScrape.getHostName(url),
searchText: SiteScrape.buildSearchText({ title, description, siteName, keywords }),
}
}
//
// Finds all urls in text, removes duplicates, makes sure they have https://
//
// DELIBERATELY UNCHANGED. This decides which URLs are detected in note text and
// what string lands in attachment.url, so altering it would change link
// behaviour for every existing note - explicitly out of scope for the scraper
// rewrite. The @TODO below is left in place for the same reason.
//
SiteScrape.getCleanUrls = (textBlock) => {
//Find all URLs in text
//@TODO - Use the process text library for this function
const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
let allUrls = textBlock.match(urlPattern)
if(allUrls == null){
return []
}
//Every URL needs HTTPS!!!
let foundUrls = []
allUrls.forEach( (item, index) => {
//add protocol if it is missing
if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
allUrls[index] = 'https://'+item
}
//convert http to https
if(item.indexOf('http://') >= 0){
allUrls[index] = item.replace('http://','https://')
}
})
//Remove all duplicates
foundUrls = [...new Set(allUrls)]
return foundUrls
}
server/index.js
+44
View File
@@ -345,10 +345,54 @@ app.use('/api/quick-note', quickNote)
var metricTracking = require('@routes/metrictrackingController')
app.use('/api/metric-tracking', metricTracking)
//
// Scraper startup.
//
// ensureScrapeSchema() converges the scrape_cache table and the attachment
// lifecycle columns. It is idempotent and runs before the listener binds, so
// there is no window where requests arrive against a schema that is mid-change.
// There is no migration framework in this project - see server/helpers/Schema.js.
//
const Schema = require('@helpers/Schema')
const ScrapeQueue = require('@workers/ScrapeQueue')
Schema.ensureScrapeSchema()
.then(() => {
ScrapeQueue.start()
// The queue lives in memory, so a pm2 reload loses whatever was in it.
// Every enqueued URL already has an attachment row, which makes the
// work re-derivable without any extra table.
return ScrapeQueue.recoverPending()
})
.then(recovery => {
if (recovery && recovery.requeued) {
console.log(`Scraper: re-queued ${recovery.requeued} pending URL(s), retired ${recovery.retired}`)
}
})
.catch(error => {
// A scraper that cannot start must not stop the API from serving.
console.log('Scraper: startup failed, link previews disabled this boot:', error.message)
})
.finally(() => {
//Output running status
app.listen(ports.express, () => {
console.log(`Express: Listening on port ${ports.express}!`)
})
})
//
// pm2 reload sends SIGINT. Stop accepting new scrape work and let in-flight
// requests abort via their own AbortControllers; rows left mid-fetch are
// recovered on the next boot.
//
const shutdown = (signal) => {
console.log(`Received ${signal}, stopping scrape queue`)
try { ScrapeQueue.stop() } catch (error) { /* already down */ }
process.exit(0)
}
process.on('SIGINT', () => shutdown('SIGINT'))
process.on('SIGTERM', () => shutdown('SIGTERM'))
//
//Error handlers
server/models/Attachment.js
+224 -233
View File
@@ -6,15 +6,30 @@ const cs = require('@helpers/CryptoString')
let Attachment = module.exports = {}
const cheerio = require('cheerio')
const rp = require('request-promise')
const request = require('request')
const fs = require('fs')
const path = require('path')
const crypto = require('crypto')
const gm = require('gm')
const tesseract = require("node-tesseract-ocr")
// Left as-is deliberately: the upload / OCR / thumbnail code below resolves
// against it, and it is correct under pm2 (cwd = server/). Not touching it
// keeps the upload path byte-identical.
const filePath = '../staticFiles/'
// Absolute equivalent, used by the scrape path only. filePath is CWD-relative,
// so under Jest (cwd = repo root) scrape downloads landed in a sibling of the
// repo rather than in staticFiles/. This resolves to the same directory as
// filePath under pm2.
const scrapeFilePath = path.join(__dirname, '..', '..', 'staticFiles') + path.sep
const config = require('@config/scrape')
const UrlSafety = require('@helpers/UrlSafety')
const HttpFetch = require('@helpers/HttpFetch')
const Realtime = require('@helpers/Realtime')
// Attachment.migrateOld
Attachment.textSearch = (userId, searchTerm) => {
@@ -263,262 +278,220 @@ Attachment.generateThumbnail = (fileName) => {
})
}
//Scans text for websites, returns all attachments
Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
return new Promise((resolve, reject) => {
//
// Diff the URLs currently in a note against the attachments already stored for
// it, soft-delete the ones that vanished, and enqueue the new ones.
//
// Returns immediately with the text already known from cache; freshly scraped
// text arrives asynchronously via the worker. Nothing here waits on the network.
//
Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
let solrAttachmentText = '' //Final searchable scrape text for note
const ScrapeQueue = require('@workers/ScrapeQueue')
if(noteText.length == 0){ resolve(solrAttachmentText) }
// Fixed: the old version called resolve() without returning, so an empty
// note fell through and ran the DB lookup anyway.
if (!noteText || noteText.length == 0) { return Promise.resolve('') }
Attachment.urlForNote(userId, noteId).then(attachments => {
let knownText = ''
//Pull all the URLs out of the text
let foundUrls = SiteScrape.getCleanUrls(noteText)
return Attachment.urlForNote(userId, noteId)
.then(attachments => {
const foundUrls = SiteScrape.getCleanUrls(noteText)
const remaining = foundUrls.slice()
//Go through each saved URL, remove new URLs from saved URLs
//If a URL is not found, delete it
attachments.forEach(attachment => {
//URL already scraped, push text and continue
let urlIndex = foundUrls.indexOf( attachment.url )
if(urlIndex != -1){
solrAttachmentText += attachment.text
foundUrls.splice(urlIndex, 1) //Remove existing from set of found
// Compared on normalised keys rather than raw strings, so an
// http/https or www. variant is not treated as a different
// resource and needlessly deleted and re-scraped.
const index = remaining.findIndex(url =>
url === attachment.url || UrlSafety.sameResource(url, attachment.url))
if (index !== -1) {
knownText += attachment.text || ''
remaining.splice(index, 1)
} else {
//If existing attachment is not found in note, remove it
// Unchanged behaviour: link attachments are hidden, not deleted.
Attachment.delete(userId, attachment.id, true)
}
})
//No newly scraped URLs, resolve with looked up attachment text
if(foundUrls == null || foundUrls.length == 0){
return resolve(solrAttachmentText)
}
if (!remaining.length) { return knownText }
//Process the remaining URLs into attachments
Attachment.scrapeUrlsCreateAttachments(userId, noteId, foundUrls).then( freshlyScrapedText => {
//Once everything is done being scraped, emit new attachment events
SocketIo.to(`user:${userId}`).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(`user:${userId}`).emit('update_note_attachments')
solrAttachmentText += freshlyScrapedText
resolve(solrAttachmentText)
// Create a shell row per new URL, then hand off to the worker.
return Promise.all(remaining.map(url =>
Attachment.createPendingUrlAttachment(userId, noteId, url)
.then(attachmentId => ({ userId, noteId, attachmentId, url }))
.catch(error => {
console.log('[scrape] could not create attachment row', url, error.message)
return null
})
.catch(console.log)
))
.then(jobs => {
const valid = jobs.filter(Boolean)
// Fire and forget - the note save must not wait on this.
ScrapeQueue.enqueueUrls(valid)
.catch(error => console.log('[scrape] enqueue failed', error.message))
return knownText
})
})
// The old version had no catch here at all.
.catch(error => {
console.log('[scrape] scanTextForWebsites failed', error.message)
return knownText
})
}
//Return scraped text from each URL
//
// Create the shell attachment row for a URL and return its id.
//
// Returns the id BEFORE any network work begins. The old code armed a 5s
// setTimeout calling request.cancel() synchronously, before this INSERT had
// resolved - so a slow INSERT meant the timer fired against a null reference
// and threw inside a bare timer callback, taking the process down.
//
Attachment.createPendingUrlAttachment = (userId, noteId, url) => {
const created = Math.round((+new Date) / 1000)
return db.promise()
.query(
`INSERT INTO attachment
(note_id, user_id, attachment_type, text, url, last_indexed, file_location, scrape_state, enqueued_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[noteId, userId, 1, url, url, created, null, 0, created]
)
.then(([result]) => result.insertId)
}
//
// Scrape a set of URLs for a note and resolve with their combined text.
//
// Kept for the bookmarklet path. The old implementation resolved only when a
// completion counter reached foundUrls.length, so a single rejection meant the
// promise never settled and the caller hung forever. allSettled inside the
// worker removes the counter entirely.
//
Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
return new Promise((resolve, reject) => {
if(foundUrls == null || foundUrls.length == 0){
return resolve('')
const ScrapeQueue = require('@workers/ScrapeQueue')
if (!foundUrls || !foundUrls.length) { return Promise.resolve('') }
return Promise.all(foundUrls.map(url =>
Attachment.createPendingUrlAttachment(userId, noteId, url)
.then(attachmentId => ({ userId, noteId, attachmentId, url }))
.catch(error => {
console.log('[scrape] could not create attachment row', url, error.message)
return null
})
))
.then(jobs => ScrapeQueue.enqueueUrls(jobs.filter(Boolean)))
.then(texts => texts.join(''))
}
console.log('About to scrape')
console.log(foundUrls)
//
// Download and store a preview image.
//
// The old downloadFileFromUrl had no timeout, no size limit, no User-Agent and
// no status check: it piped whatever came back straight into a write stream, so
// a 404 HTML error page was written to disk and handed to gm. It also wrote the
// file before validating anything, which is why staticFiles/ accumulated
// zero-byte orphans.
//
Attachment.downloadThumbnail = (imageUrl, pageUrl) => {
let processedCount = 0
let scrapedText = ''
if (!imageUrl) { return Promise.resolve(null) }
//Process each URL passd to function, a DB entry will be created for each scrape
foundUrls.forEach(url => {
Attachment.processUrl(userId, noteId, url).then( freshlyScrapedText => {
// crypto.randomBytes rather than two concatenated Math.random() calls.
const fileName = crypto.randomBytes(16).toString('hex') + '_scrape'
const fullPath = scrapeFilePath + fileName
const thumbPath = scrapeFilePath + 'thumb_' + fileName
const tempPath = fullPath + '.tmp'
scrapedText += freshlyScrapedText
processedCount ++
return HttpFetch.fetchBinary(imageUrl, {
pageUrl,
timeoutMs: config.imageTimeoutMs,
maxBytes: config.maxImageBytes,
})
.then(result => {
//All URLs have been scraped, return data
if(processedCount == foundUrls.length){
console.log('All urls scraped')
return resolve(scrapedText)
// Status and content-type were both gated inside fetchBinary, so by
// here we are holding bytes that really are an image.
if (!result.buffer || !result.buffer.length) { return null }
if (!magicBytesMatch(result.buffer, result.mime)) {
console.log('[scrape] image bytes do not match declared type', result.mime, imageUrl)
return null
}
// Write to a temp path, then rename, so a failure cannot leave a
// half-written file behind for the client to serve.
return fs.promises.writeFile(tempPath, result.buffer)
.then(() => fs.promises.rename(tempPath, fullPath))
.then(() => resizeToThumbnail(fullPath, thumbPath))
.then(() => fileName)
.catch(error => {
console.log('[scrape] could not store image', imageUrl, error.message)
return cleanupFiles([tempPath, fullPath, thumbPath]).then(() => null)
})
})
.catch(error => {
console.log('Site Scrape error', error)
})
})
console.log('[scrape] image fetch failed', imageUrl, error.kind || error.message)
return null
})
}
Attachment.downloadFileFromUrl = (url) => {
return new Promise((resolve, reject) => {
if(!url){
return resolve(null)
}
const random = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
let extension = ''
let fileName = random+'_scrape'
let thumbPath = 'thumb_'+fileName
console.log('Scraping image url', url)
console.log('Getting ready to scrape ', url)
request(url)
.on('error', error => {
console.log(error)
resolve(null)
})
.on('response', res => {
console.log(res.statusCode)
console.log(res.headers['content-type'])
//Get mime type from header content type
// extension = '.'+String(res.headers['content-type']).split('/').pop()
})
.pipe(fs.createWriteStream(filePath+thumbPath))
.on('close', () => {
//resize image if its real big
gm(filePath+thumbPath)
//
// Resize via gm. Degrades rather than failing: gm (and tesseract) are not in
// shell.nix buildInputs, so the binary is absent in dev - the old code returned
// null here and left the downloaded file orphaned on disk. Copying the
// full-size image to the thumbnail path keeps the preview working.
//
const resizeToThumbnail = (fullPath, thumbPath) => {
return new Promise(resolve => {
gm(fullPath)
.resize(550) //Resize to width of 550 px
.quality(85) //compression level 0 - 100 (best)
.write(filePath+thumbPath, function (err) {
if(err){
console.log(err)
return resolve(null)
}
console.log('Saved Image')
return resolve(fileName)
})
.write(thumbPath, error => {
if (!error) { return resolve(true) }
console.log('[scrape] gm unavailable or failed, using full size as thumbnail')
fs.promises.copyFile(fullPath, thumbPath)
.then(() => resolve(true))
.catch(() => resolve(false))
})
})
}
Attachment.processUrl = (userId, noteId, url) => {
const scrapeTime = 5*1000;
return new Promise((resolve, reject) => {
const options = {
uri: url,
simple: true,
timeout: scrapeTime,
headers: {
'User-Agent':'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' //Simulate google headers
},
transform: function (body) {
return cheerio.load(body);
}
const cleanupFiles = (paths) => {
return Promise.all(paths.map(target =>
fs.promises.unlink(target).catch(() => null)
))
}
let requestTimeout = null
let thumbnail = null
let request = null
let created = Math.round((+new Date)/1000)
let insertedId = null
//
// Sanity-check the bytes against the declared content type. A server can claim
// image/png and send anything.
//
const IMAGE_MAGIC = [
{ mime: 'image/png', bytes: [0x89, 0x50, 0x4e, 0x47] },
{ mime: 'image/jpeg', bytes: [0xff, 0xd8, 0xff] },
{ mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] },
{ mime: 'image/bmp', bytes: [0x42, 0x4d] },
]
//Create a shell attachment for each URL, put in processing state
db.promise()
.query(`INSERT INTO attachment
(note_id, user_id, attachment_type, text, url, last_indexed, file_location)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
[noteId, userId, 1, url, url, created, null])
.then((rows, fields) => {
//Set two bigger variables then return request for processing
request = rp(options)
insertedId = rows[0].insertId
const magicBytesMatch = (buffer, mime) => {
const expected = IMAGE_MAGIC.find(entry => entry.mime === mime)
return request
})
.then($ => {
// webp/avif/ico and friends have container-dependent headers - accept them
// on the content-type alone rather than guessing wrong.
if (!expected) { return true }
if (buffer.length < expected.bytes.length) { return false }
//Clear timeout that would end this function
clearTimeout(requestTimeout)
// let header = $('h1').text().replace(removeWhitespace, " ")
// desiredSearchText += header + "\n"
const pageTitle = SiteScrape.getTitle($)
const hostname = SiteScrape.getHostName(url)
const thumbnail = SiteScrape.getDisplayImage($, url)
const keywords = SiteScrape.getKeywords($)
var desiredSearchText = ''
desiredSearchText += pageTitle
if(keywords){
desiredSearchText += "\n " + keywords
}
console.log('Results from site scrape-------------')
console.log({
pageTitle,
hostname,
thumbnail,
keywords
})
// throw new Error('Ending this function early.')
// console.log('TexT Scraped')
// console.log(desiredSearchText)
created = Math.round((+new Date)/1000)
//Scrape URL for thumbnail - take filename and save in attachment
Attachment.downloadFileFromUrl(thumbnail)
.then(thumbnailFilename => {
//Update text and thumbnail filename
created = Math.round((+new Date)/1000)
db.promise()
.query(`UPDATE attachment SET
text = ?,
last_indexed = ?,
file_location = ?
WHERE id = ?
`, [desiredSearchText, created, thumbnailFilename, insertedId])
.then((rows, fields) => {
resolve(desiredSearchText) //Return found text
})
.catch(console.log)
//Create attachment in DB with scrape text and provided data
// db.promise()
// .query(`INSERT INTO attachment
// (note_id, user_id, attachment_type, text, url, last_indexed, file_location)
// VALUES (?, ?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created, thumbnailFilename])
// .then((rows, fields) => {
// resolve(desiredSearchText) //Return found text
// })
// .catch(console.log)
})
})
.catch(error => {
console.log('Scrape pooped out')
console.log('Issue with scrape', error.statusCode)
clearTimeout(requestTimeout)
return resolve('No site text')
})
requestTimeout = setTimeout( () => {
console.log('Cancel the request, its taking to long.')
request.cancel()
return resolve('Request Timeout')
}, scrapeTime )
})
return expected.bytes.every((byte, index) => buffer[index] === byte)
}
Attachment.generatePushKey = (userId) => {
@@ -576,36 +549,54 @@ Attachment.getPushkeyBookmarklet = (userId) => {
})
}
//
// Bookmarklet entry point: GET /api/public/pushmebaby?pushkey=...&url=...
//
// The URL arrives straight off a query string, so it is validated here - the
// old code passed it through completely unvalidated, bypassing even the https
// normalisation that note text goes through.
//
// Response timing is preserved: the bookmarklet still answers after the scrape,
// but now with a hard cap instead of the previous unbounded hang.
//
Attachment.pushUrl = (pushkey, url) => {
return new Promise((resolve, reject) => {
const ScrapeQueue = require('@workers/ScrapeQueue')
let userId = null
pushkey = pushkey.replace(/ /g, '+')
pushkey = String(pushkey || '').replace(/ /g, '+')
db.promise()
return db.promise()
.query("SELECT id FROM user WHERE pushkey = ? LIMIT 1", [pushkey])
.then((rows, fields) => {
.then(([rows]) => {
if(rows[0].length == 0){
return resolve(true)
}
// Unchanged: an unknown pushkey resolves truthy and leaks nothing
// about whether the key exists.
if (rows.length == 0) { return null }
userId = rows[0][0].id
return Attachment.scrapeUrlsCreateAttachments(userId, null, [url])
userId = rows[0].id
return UrlSafety.assertFetchable(url)
.then(() => Attachment.createPendingUrlAttachment(userId, null, url))
.then(attachmentId => ScrapeQueue.awaitUrl(
ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url }),
config.totalBudgetMs
))
.catch(error => {
console.log('[scrape] pushUrl refused', url, error.reason || error.message)
return null
})
})
.then(() => {
if(typeof SocketIo != 'undefined'){
//Once everything is done being scraped, emit new attachment events
SocketIo.to(`user:${userId}`).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(`user:${userId}`).emit('update_note_attachments')
if (userId) {
Realtime.emitToUser(userId, 'update_counts')
Realtime.emitToUser(userId, 'update_note_attachments')
}
return resolve(true)
return true
})
.catch(console.log)
.catch(error => {
console.log('[scrape] pushUrl failed', error.message)
return true
})
}
server/models/Note.js
+8 -5
View File
@@ -4,6 +4,7 @@ let Note = module.exports = {}
let Tags = require('@models/Tag')
let Attachment = require('@models/Attachment')
let Realtime = require('@helpers/Realtime')
let ShareNote = require('@models/ShareNote')
let ProcessText = require('@helpers/ProcessText')
@@ -12,7 +13,6 @@ const DiffMatchPatch = require('@helpers/DiffMatchPatch')
const crypto = require('crypto')
const cs = require('@helpers/CryptoString')
const rp = require('request-promise');
const fs = require('fs')
const gm = require('gm')
@@ -513,12 +513,15 @@ Note.update = (userId, noteId, noteText, noteTitle, color, pinned, archived, has
})
.then((rows, fields) => {
if(typeof SocketIo != 'undefined'){
SocketIo.to(`user:${userId}`).emit('new_note_text_saved', {noteId, hash})
// The old `if(typeof SocketIo != 'undefined')` guard did not just
// skip the emit - it skipped the scrape too, so link scraping was
// silently disabled under Jest and could never be tested. Realtime
// no-ops safely without a socket server, so both now always run.
Realtime.emitToUser(userId, 'new_note_text_saved', {noteId, hash})
//Async attachment reindex
Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
}
Attachment.scanTextForWebsites(userId, noteId, noteText)
.catch(error => console.log('scanTextForWebsites failed', error.message))
//Send back updated response
resolve(rows[0])
server/tests/models/Attachment.test.js
+118 -9
View File
@@ -1,5 +1,11 @@
const Attachment = require('../../models/Attachment')
const User = require('../../models/User')
const ScrapeQueue = require('@workers/ScrapeQueue')
const Robots = require('@helpers/Robots')
const FetchMock = require('../support/fetchMock')
const dns = require('dns').promises
let dnsSpy = null
const testUserName = 'jestTestUserAttachment'
const password = 'Beans19934!!!'
@@ -67,19 +73,122 @@ test('Test get Push Key Bookmarklet', () => {
})
test('Test Push URL', () => {
//
// These used to hit https://www.solidscribe.com over the live network, so the
// suite needed internet access and the assertion depended on whatever that page
// happened to serve. fetch is mocked against a fixture instead.
//
describe('Push URL', () => {
let url = 'https://www.solidscribe.com'
beforeEach(() => {
FetchMock.install()
Robots.__resetCache()
dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
// Real code path, no politeness waiting and no robots lookups.
ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false })
})
afterEach(() => {
FetchMock.restore()
dnsSpy.mockRestore()
ScrapeQueue.__reset()
})
test('scrapes a pushed url into a searchable attachment', () => {
const url = 'https://www.solidscribe.com/'
FetchMock.route(url, () => FetchMock.html(`
<html><head>
<title>Solid Scribe</title>
<meta property="og:description" content="An encrypted note taking application.">
</head><body><p>x</p></body></html>
`))
return Attachment.pushUrl(newPushKey, url)
.then(( results => {
return Attachment.textSearch(newUserId, 'scribe')
}))
.then((results) => {
.then(() => ScrapeQueue.drain())
.then(() => Attachment.textSearch(newUserId, 'scribe'))
.then(results => {
expect(results.length == 1).toBe(true)
expect(results[0].text).toContain('Solid Scribe')
})
})
test('stores the description so it is searchable too', () => {
const url = 'https://www.solidscribe.com/encrypted'
FetchMock.route(url, () => FetchMock.html(`
<html><head>
<title>Encryption Page</title>
<meta property="og:description" content="Notes are sealed with aes256gcm.">
</head><body><p>x</p></body></html>
`))
return Attachment.pushUrl(newPushKey, url)
.then(() => ScrapeQueue.drain())
.then(() => Attachment.textSearch(newUserId, 'aes256gcm'))
.then(results => {
expect(results.length).toBeGreaterThan(0)
})
})
// The pushmebaby query param was previously passed through with no
// validation whatsoever.
test('refuses a pushed url pointing at loopback and makes no request', () => {
FetchMock.route(/.*/, () => FetchMock.html('<title>should never load</title>'))
return Attachment.pushUrl(newPushKey, 'http://127.0.0.1:3000/admin')
.then(result => {
expect(result).toBe(true)
expect(FetchMock.callCount()).toBe(0)
})
})
test('refuses a pushed url pointing at the cloud metadata endpoint', () => {
FetchMock.route(/.*/, () => FetchMock.html('<title>should never load</title>'))
return Attachment.pushUrl(newPushKey, 'http://169.254.169.254/latest/meta-data/')
.then(() => {
expect(FetchMock.callCount()).toBe(0)
})
})
test('resolves truthy for an unknown pushkey without scraping', () => {
FetchMock.route(/.*/, () => FetchMock.html('<title>nope</title>'))
return Attachment.pushUrl('not-a-real-pushkey', 'https://example.com/')
.then(result => {
expect(result).toBe(true)
expect(FetchMock.callCount()).toBe(0)
})
})
// Previously this could hang the HTTP response indefinitely.
test('returns within the budget when the server never responds', () => {
const url = 'https://www.solidscribe.com/slow'
FetchMock.route(url, () => FetchMock.hang())
ScrapeQueue.configure({
perHostMinSpacingMs: 0,
perHostMaxSpacingMs: 0,
checkRobots: false,
pageTimeoutMs: 80,
totalBudgetMs: 300,
retries: 0,
})
const startedAt = Date.now()
return Attachment.pushUrl(newPushKey, url)
.then(result => {
expect(result).toBe(true)
expect(Date.now() - startedAt).toBeLessThan(5000)
})
})
})