Max/SolidScribe
1
0 Fork 0
Code Issues 24 Pull Requests Releases Wiki Activity

Compare commits

base: Max/SolidScribe:590ae01055cd396ecccf734a2db6e69b202b8649
Max/SolidScribe:dev Max/SolidScribe:vue3_vite_tailwind_update
..
compare: Max/SolidScribe:vue3_vite_tailwind_update
Max/SolidScribe:vue3_vite_tailwind_update Max/SolidScribe:dev
12 Commits
590ae01055 .. vue3_vite_tailwind_update
Author SHA1 Message Date
Max 1b47cb0d35 Testing Migrations for web scraper 2026-11-03 13:03:45 -07:00
Max 9804771a5b Testing Migrations 2026-11-03 12:53:17 -07:00
Max d9abeb7fb1 Testing Migrations 2026-11-03 12:50:32 -07:00
Max 81b4162117 Deploy and schema updates for testing 2026-11-03 11:48:06 -07:00
Max 8d60aae65c Adding scrape files for testing 2026-11-03 11:20:03 -07:00
Max 7d255fe687 Small update to style for icons. tweaked pm2 runner to work with deploy script 2026-11-03 11:11:19 -07:00
max b33e2710c3 Update ecosystem for pm2 2026-11-03 17:57:52 +00:00
Max 26e31f1885 Scraper Update for testing 2026-11-03 10:23:03 -07:00
Max 7f38444707 Update host dir and release process 2026-10-29 12:52:50 -07:00
Max 2ce1fc073d Update host dir and release process 2026-10-28 16:30:36 -07:00
Max 42287395fe Big update to deploy scripts, and deploy prosecss 2026-10-28 12:17:09 -07:00
Max 98261207d6 Updated Version 2026-10-28 12:07:03 -07:00
63 changed files with 7348 additions and 1134 deletions
Expand all files Collapse all files

No files matched your search

client/src/assets/semantic-helper.css
+1 -1
View File
@@ -179,7 +179,7 @@ div.ui.basic.green.label {
}
/*Overwrites for modifiable theme color */
i.green.icon.icon.icon.icon, i.green.icon.icon.icon.icon.icon {
color: var(--main-accent);
color: var(--main-accent) !important;
}
.button {
box-shadow: 2px 2px 4px -2px rgba(40, 40, 40, 0.89) !important;
client/vite.config.js
+23 -7
View File
@@ -66,13 +66,29 @@ export default defineConfig({
// mid-session underneath them.
clientsClaim: false,
skipWaiting: false,
runtimeCaching: [
{
urlPattern: ({ url }) =>
url.pathname.startsWith('/api/') || url.pathname.startsWith('/socket'),
handler: 'NetworkOnly',
},
],
//
// runtimeCaching is deliberately EMPTY. Do not add an entry for /api/*
// or /socket -- not even NetworkOnly.
//
// There used to be a NetworkOnly route for both, as belt-and-braces for
// the invariant above. It was worse than useless. registerRoute() makes
// the worker call event.respondWith() for every matching request, and
// Socket.IO's long-poll requests are aborted as a matter of course --
// on reconnect, on transport upgrade, and whenever Android backgrounds
// the page. An aborted fetch inside NetworkOnly throws Workbox's
// `no-response`, the respondWith promise rejects, and Socket.IO sees a
// dead transport and reconnects. The installed PWA was dropping its
// live note sync on a loop and filling the logs with
// "FetchEvent ... resulted in a network error response".
//
// With no route registered the worker never intercepts these at all and
// the browser handles them natively, exactly as in a normal tab.
//
// Nothing is lost on the security side: the invariant is enforced by
// globPatterns only ever reading dist/, and by there being no *caching*
// strategy here. NetworkOnly enforced nothing -- it only inserted the
// worker into the hot path of every poll.
//
},
// Flip to true temporarily to exercise the worker on the dev server.
// Remember to unregister it afterwards -- a lingering dev service worker
configs/nginx-nixos.nix
+18 -5
View File
@@ -17,6 +17,13 @@
# /assets/, /sw.js and /manifest.json unresolvable once they needed their
# own location blocks.
#
# It also no longer points into the repo. deploy.sh publishes the built
# client to /var/www/ss/releases/<timestamp> and atomically repoints the
# /var/www/ss/current symlink at it. Pointing nginx at that stable path
# means a deploy is one rename -- nginx resolves the symlink per request,
# so no reload is needed and no request sees a half-copied directory.
# Rollback is the same rename pointing at an older release.
#
# 3. proxyWebsockets on /socket. recommendedProxySettings does NOT add the
# Upgrade/Connection headers, so Socket.IO has been silently falling back
# to long-polling -- that's the live note-sync path running degraded.
@@ -45,7 +52,8 @@ services.nginx = {
virtualHosts."192.168.1.7" = {
# serverAliases = [ "notes.example.com" ]; # if the tunnel forwards the public host
root = "REPO_PATH/client/dist";
# Published by deploy.sh; a symlink to the live release. Not in the repo.
root = "/var/www/ss/current";
locations."/" = {
tryFiles = "$uri $uri/ /index.html";
@@ -84,7 +92,12 @@ services.nginx = {
};
};
# Note on file access: nginx runs as its own user, so REPO_PATH/client/dist must
# be readable by it -- every parent directory needs +x. If the repo lives under
# /home/mab this usually means loosening the home directory's permissions, or
# putting the clone somewhere like /srv instead.
# Note on file access: nginx runs as its own user and needs read access to the
# web root, with +x on every parent directory. Serving from /var/www/ss is what
# makes that a non-issue -- the one-time `chmod 755 /var/www /var/www/ss` in
# deploy.sh is the whole of it. This is the reason the root is not the repo
# clone: a clone under /home/mab would require loosening the permissions on the
# home directory itself.
#
# Uploads are NOT here. Express serves them from REPO_PATH/staticFiles at
# /api/static via the proxy below, so they are untouched by deploys.
deploy.sh
+154 -15
View File
@@ -13,24 +13,126 @@
# environment.systemPackages or the user's profile -- NOT only inside a
# nix-shell, or this will fail at the install step)
# - pm2 is already managing NoteServer (pm2 resurrect on boot)
# - nginx serves client/dist with an SPA fallback to index.html
# - nginx serves $WEBROOT/current with an SPA fallback to index.html
#
# nginx is NOT touched here. On NixOS it is declarative -- see configs/.
#
# --- the web root --------------------------------------------------------
#
# The built client is published to /var/www/ss, NOT served out of the repo.
# nginx runs as its own user, so serving from a clone under /home/mab meant
# loosening the home directory's permissions for every parent dir. /var/www is
# the conventional place for this and sidesteps the whole problem.
#
# Layout, which is the standard symlink-swap pattern:
#
# /var/www/ss/
# releases/2026-10-28_1147/ <- one directory per deploy
# releases/2026-10-28_0930/
# current -> releases/2026-10-28_1147
#
# nginx points at the stable path $WEBROOT/current. Publishing is a rename of
# that symlink, which is a single atomic syscall: no request ever sees a
# half-copied directory, and no window exists where the root is missing.
# Rolling back is the same rename pointing at the previous release.
#
# Only the *client* moves. The API server still runs from the repo under pm2,
# and user uploads are still served by Express from $REPO/staticFiles at
# /api/static -- deliberately, so deploys never touch user data.
#
# ONE-TIME SETUP on a new server (needs root):
#
# sudo mkdir -p /var/www/ss/releases
# sudo chown -R "$USER" /var/www/ss
# sudo chmod 755 /var/www /var/www/ss
#
# Override the location for a staging box with: WEBROOT=/var/www/ss-staging ./deploy.sh
#
# --- the database account ------------------------------------------------
#
# The API connects as a least-privilege account with DML rights only. This is
# deliberate: the server fetches untrusted URLs for link previews, and a
# service able to reshape or drop its own tables is a far larger blast radius
# than that feature warrants.
#
# CREATE USER 'app'@'localhost' IDENTIFIED BY '<password>';
# GRANT SELECT, INSERT, UPDATE, DELETE ON application.* TO 'app'@'localhost';
# FLUSH PRIVILEGES;
#
# -- deliberately NOT granted: CREATE, ALTER, DROP, INDEX
#
# Then set DB_USER=app and DB_PASS=<password> in ~/.env. Note the asymmetry:
# the admin scripts here (backupDatabase.sh, applyLastBackup.sh,
# migrateDatabase.sh) still connect as root with $PROD_DB_PASS.
#
# DELETE is genuinely required -- the scraper drops shell attachment rows for
# dead links and purges expired cache entries.
#
# --- schema changes ------------------------------------------------------
#
# Schema is managed by Prisma, for migrations and scaffolding ONLY -- the
# running server never imports it and still talks to MariaDB through raw SQL.
# The schema lives in prisma/schema.prisma; migrations in prisma/migrations/.
#
# Because the app holds no DDL, a deploy that introduces schema changes needs
# one manual step, run once per environment as a database admin:
#
# nix-shell shell.nix --run ./migrateDatabase.sh
#
# nix-shell is required: the prisma CLI comes from nixpkgs (prisma_7, 7.8.0),
# not from node_modules. Alternatively add prisma_7 to the server's
# environment.systemPackages and call the script directly.
#
# It is idempotent, so re-running is safe. On a database that predates Prisma
# the script detects the missing migration history and baselines it
# automatically (marking 0_init applied rather than executing it) -- it does
# NOT need to be done by hand.
#
# The API verifies the schema and the grants at boot; if either is missing it
# logs the remedy and serves normally with link previews disabled.
#
# NEVER run `prisma migrate dev`, `prisma migrate reset` or `prisma db push`.
# The first wants a shadow database; the latter two can drop data.
#
# DATABASE_URL is composed by the script from the admin credentials in ~/.env
# and is never read by the application.
#
# Note: _prisma_migrations lives inside the `application` database, so the
# least-privilege `app` user can write migration history (its grant is
# database-wide). It holds no DDL, so it cannot reshape tables -- narrowing
# this further would mean per-table grants, which is a separate project.
#
set -euo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$REPO"
WEBROOT="${WEBROOT:-/var/www/ss}"
RELEASE="$WEBROOT/releases/$(date +%Y-%m-%d_%H%M%S)"
# Number of old releases to keep around for rollback.
KEEP_RELEASES=5
step(){ printf '\n\033[32m::--:: %s\033[0m\n' "$1"; }
# --- sanity checks before changing anything ------------------------------
for tool in git pnpm pm2; do
for tool in git pnpm pm2 rsync; do
command -v "$tool" >/dev/null || { echo "$tool is not on PATH" >&2; exit 1; }
done
# Fail here rather than after a three-minute build. A missing web root is
# almost always a server that never had the one-time setup above run on it.
if [ ! -d "$WEBROOT/releases" ]; then
echo "$WEBROOT/releases does not exist -- run the one-time setup at the top of this script." >&2
exit 1
fi
if [ ! -w "$WEBROOT/releases" ]; then
echo "$WEBROOT/releases is not writable by $(whoami) -- check the chown in the one-time setup." >&2
exit 1
fi
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
echo "Working tree has local modifications. Commit or stash them first:" >&2
git status --short >&2
@@ -54,23 +156,54 @@ pnpm install --frozen-lockfile --dir client
# --- build ---------------------------------------------------------------
# dist is gitignored, so the server builds it. Built into a temp dir first so a
# failed build leaves the currently-served files untouched.
# dist is gitignored, so the server builds it. The build stays inside the repo
# and is only copied to $WEBROOT once it has been checked, so a failed build
# cannot touch what is currently being served.
step "Building client..."
rm -rf client/dist.new
pnpm --dir client exec vite build --outDir dist.new --emptyOutDir
rm -rf client/dist
pnpm --dir client exec vite build
if [ ! -f client/dist.new/index.html ] || [ ! -f client/dist.new/sw.js ]; then
echo "Build output looks wrong (missing index.html or sw.js) -- keeping the previous build." >&2
rm -rf client/dist.new
# manifest.json is checked in under client/public and copied verbatim by Vite;
# if it is missing, the PWA installs without icons, scope or a share target.
for required in index.html sw.js manifest.json; do
if [ ! -f "client/dist/$required" ]; then
echo "Build output looks wrong (missing $required) -- $WEBROOT left untouched." >&2
exit 1
fi
fi
done
step "Swapping in the new build..."
rm -rf client/dist.old
[ -d client/dist ] && mv client/dist client/dist.old
mv client/dist.new client/dist
rm -rf client/dist.old
# --- publish -------------------------------------------------------------
step "Publishing to $RELEASE..."
mkdir -p "$RELEASE"
# Trailing slash on the source: copy the *contents* of dist into the release
# dir. --delete is harmless on a fresh dir but keeps a re-run of the same
# timestamp honest.
rsync -a --delete client/dist/ "$RELEASE/"
# nginx reads these as its own user, so they must be world-readable.
chmod -R a+rX "$RELEASE"
step "Pointing $WEBROOT/current at the new release..."
# Two steps because `ln -sfn` over an existing symlink unlinks before relinking,
# which leaves a brief window with no web root at all. Creating a scratch
# symlink and renaming it over the old one is a single atomic rename(2).
ln -sfn "$RELEASE" "$WEBROOT/current.new"
mv -T "$WEBROOT/current.new" "$WEBROOT/current"
# --- prune ---------------------------------------------------------------
# Keep the most recent $KEEP_RELEASES for rollback, drop the rest. Never remove
# whatever `current` actually resolves to, however old it is.
live="$(readlink -f "$WEBROOT/current")"
ls -1d "$WEBROOT"/releases/*/ 2>/dev/null \
| sort -r \
| tail -n +$((KEEP_RELEASES + 1)) \
| while read -r stale; do
[ "$(readlink -f "$stale")" = "$live" ] && continue
echo " pruning $(basename "$stale")"
rm -rf "$stale"
done
# --- restart -------------------------------------------------------------
@@ -83,4 +216,10 @@ pm2 reload "$REPO/server/ecosystem.config.js" --env production --update-env
pm2 save
step "Done."
echo " serving $(readlink -f "$WEBROOT/current")"
echo
echo " To roll back, repoint the symlink at the previous release:"
echo " ls $WEBROOT/releases"
echo " ln -sfn $WEBROOT/releases/<older> $WEBROOT/current.new && mv -T $WEBROOT/current.new $WEBROOT/current"
echo
pm2 list
jest.config.js
+15 -7
View File
@@ -1,11 +1,19 @@
const path = '../../'
const prefix = '/$1'
//
// The @alias map must mirror _moduleAliases in package.json, which is what
// module-alias/register sets up at runtime.
//
// These were previously written as "../../server/..." relative paths, which
// jest resolves relative to the requiring file - so they never actually
// resolved, and every existing test sidestepped them with relative requires.
// <rootDir> is depth-independent and works from any test directory.
//
module.exports = {
moduleNameMapper: {
"@root/(.*)": ".",
"@models/(.*)": path+"server/models"+prefix,
"@routes/(.*)": path+"server/routes"+prefix,
"@helpers/(.*)": path+"server/helpers"+prefix,
"@config/(.*)": path+"server/config"+prefix,
"^@root/(.*)$": "<rootDir>/$1",
"^@models/(.*)$": "<rootDir>/server/models/$1",
"^@routes/(.*)$": "<rootDir>/server/routes/$1",
"^@helpers/(.*)$": "<rootDir>/server/helpers/$1",
"^@config/(.*)$": "<rootDir>/server/config/$1",
"^@workers/(.*)$": "<rootDir>/server/workers/$1",
}
}
migrateDatabase.sh
Executable
+292
View File
@@ -0,0 +1,292 @@
#!/usr/bin/env bash
#
# Apply database migrations. Run from the repo root, INSIDE nix-shell (that is
# where the prisma CLI lives).
#
# nix-shell shell.nix --run ./migrateDatabase.sh
#
# Works both ways round: it applies pending migrations to an existing database,
# and builds a complete one from empty. Safe to re-run -- a second run is a
# no-op. Replaces migrateDatabase.sh.
#
# --- why this is a separate step -----------------------------------------
#
# The application connects as a least-privilege account (production uses `app`)
# holding DML rights only -- no CREATE, ALTER or DROP. That is deliberate: the
# server fetches untrusted URLs for link previews, and a service able to reshape
# its own tables is a far larger blast radius than that feature warrants.
#
# So the app never applies DDL. It verifies the schema at boot
# (server/helpers/Schema.js) and disables link previews if anything is missing.
#
# --- authoring a new migration -------------------------------------------
#
# NEVER run `prisma migrate dev`. It can decide the database has drifted and
# offer to RESET it, which on a box holding a production data restore would
# destroy real data. It also wants a shadow database. Instead:
#
# 1. Edit prisma/schema.prisma
# 2. NAME=add_whatever
# DIR="prisma/migrations/$(date +%Y%m%d%H%M%S)_$NAME"
# mkdir -p "$DIR"
# prisma migrate diff --from-config-datasource \
# --to-schema prisma/schema.prisma --script > "$DIR/migration.sql"
# 3. READ the generated SQL -- this review is what makes the approach safe
# 4. ./migrateDatabase.sh
#
# Note: Prisma emits `COLLATE utf8mb4_unicode_ci`, but this database is
# `utf8mb4_general_ci` throughout, and `prisma migrate diff` does NOT report the
# difference. Fix the collation in any generated migration by hand. This script
# checks for drift at the end.
#
# --- credentials ---------------------------------------------------------
#
# Same convention as applyLastBackup.sh / backupDatabase.sh: source ~/.env and
# connect as root with $PROD_DB_PASS. Falls back to passwordless root (the dev
# case, where shell.nix also exports DATABASE_URL). Override with:
#
# DB_ADMIN_USER=admin DB_ADMIN_PASS=secret ./migrateDatabase.sh
#
set -euo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$REPO"
DB_NAME="application" # hardcoded in server/config/database.js too
# --- preconditions -------------------------------------------------------
if ! command -v prisma >/dev/null 2>&1; then
echo "prisma is not on PATH -- run this inside nix-shell:" >&2
echo " nix-shell shell.nix --run ./migrateDatabase.sh" >&2
exit 1
fi
if ! command -v mariadb >/dev/null 2>&1 && ! command -v mysql >/dev/null 2>&1; then
echo "no mariadb/mysql client on PATH -- run this inside nix-shell." >&2
exit 1
fi
if command -v mariadb >/dev/null 2>&1; then MYSQL_BIN="mariadb"; else MYSQL_BIN="mysql"; fi
if ! command -v node >/dev/null 2>&1; then
echo "node is not on PATH -- needed to percent-encode the database password." >&2
exit 1
fi
if [ -f "$HOME/.env" ]; then
# shellcheck disable=SC1091
source "$HOME/.env"
fi
ADMIN_USER="${DB_ADMIN_USER:-root}"
ADMIN_PASS="${DB_ADMIN_PASS:-${PROD_DB_PASS:-}}"
mysql_args=(--user "$ADMIN_USER")
[ -n "$ADMIN_PASS" ] && mysql_args+=(--password="$ADMIN_PASS")
LOCAL_SOCKET="$REPO/.mysql/mysql.sock"
[ -S "$LOCAL_SOCKET" ] && mysql_args+=(--socket="$LOCAL_SOCKET")
# Prisma needs a URL, and it must carry the ADMIN credentials -- never the
# app account, which holds no DDL.
DB_ADMIN_HOST="${DB_ADMIN_HOST:-127.0.0.1}"
DB_ADMIN_PORT="${DB_ADMIN_PORT:-3306}"
#
# Percent-encode the password. Production passwords routinely contain @ : / ? #
# or %, every one of which silently corrupts a URL -- an unencoded `@` would
# truncate the host and produce a baffling connection error rather than an
# obvious one.
#
encode_component() {
node -e 'process.stdout.write(encodeURIComponent(process.argv[1] || ""))' "$1"
}
ADMIN_USER_ENC="$(encode_component "$ADMIN_USER")"
if [ -n "$ADMIN_PASS" ]; then
ADMIN_PASS_ENC="$(encode_component "$ADMIN_PASS")"
export DATABASE_URL="mysql://${ADMIN_USER_ENC}:${ADMIN_PASS_ENC}@${DB_ADMIN_HOST}:${DB_ADMIN_PORT}/${DB_NAME}"
else
export DATABASE_URL="mysql://${ADMIN_USER_ENC}@${DB_ADMIN_HOST}:${DB_ADMIN_PORT}/${DB_NAME}"
fi
# Keep the CLI from phoning checkpoint.prisma.io for update checks.
export CHECKPOINT_DISABLE=1
if ! "$MYSQL_BIN" "${mysql_args[@]}" -e "SELECT 1" >/dev/null 2>&1; then
echo "Cannot connect to the database as '$ADMIN_USER'." >&2
echo "Set PROD_DB_PASS in ~/.env, or pass DB_ADMIN_USER / DB_ADMIN_PASS." >&2
exit 1
fi
#
# Collation is explicit because the server default is utf8mb4_unicode_ci while
# every application table is utf8mb4_general_ci. A table created later without
# an explicit COLLATE would otherwise inherit the wrong one, changing string
# comparison and FULLTEXT ranking -- and Prisma neither models collation nor
# reports the difference in `migrate diff`.
#
# For a database that already exists with the wrong default, the one-time
# non-destructive fix is:
# ALTER DATABASE application CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
#
"$MYSQL_BIN" "${mysql_args[@]}" -e "CREATE DATABASE IF NOT EXISTS \`$DB_NAME\` \
CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"
sql() { "$MYSQL_BIN" "${mysql_args[@]}" -N -B "$DB_NAME" -e "$1" 2>/dev/null; }
echo '-------'
echo "Migrating '$DB_NAME' as '$ADMIN_USER'"
echo '-------'
# --- baseline an existing, pre-Prisma database ---------------------------
#
# This is the single most dangerous moment in adopting Prisma. A database that
# already has the schema but no _prisma_migrations table must have the baseline
# marked applied, NOT executed -- running 0_init against it would fail on
# existing tables. Detect and handle it here rather than leaving it as a manual
# step someone forgets.
#
HAS_MIGRATION_TABLE=$(sql "SELECT COUNT(*) FROM information_schema.TABLES
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='_prisma_migrations'" || echo 0)
HAS_EXISTING_SCHEMA=$(sql "SELECT COUNT(*) FROM information_schema.TABLES
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment'" || echo 0)
if [ "${HAS_MIGRATION_TABLE:-0}" -eq 0 ] && [ "${HAS_EXISTING_SCHEMA:-0}" -gt 0 ]; then
echo "Existing schema found with no migration history -- baselining."
#
# Report drift BEFORE recording the baseline.
#
# `migrate resolve` records a migration without executing it, so baselining
# a database that does not actually match the schema leaves it permanently
# short of those objects while Prisma reports everything as applied. That
# is exactly how production ended up missing scrape_cache: this script
# previously inferred "the attachment table exists" => "the whole schema
# exists", which is not the same claim.
#
# Later migrations are additive and idempotent, so drift is recoverable --
# but the operator should see it rather than discover it from a log line
# days later.
#
if ! prisma migrate diff \
--from-config-datasource \
--to-schema prisma/schema.prisma \
--exit-code >/dev/null 2>&1
then
echo ""
echo "NOTE: this database does not yet match prisma/schema.prisma."
echo " Baselining 0_init anyway, then applying later migrations to"
echo " converge it. Differences:"
prisma migrate diff --from-config-datasource --to-schema prisma/schema.prisma \
| sed 's/^/ /'
echo ""
fi
echo "Marking 0_init as already applied (NOT executing it)."
prisma migrate resolve --applied 0_init
echo ""
fi
# --- apply ---------------------------------------------------------------
prisma migrate deploy
# --- seed ----------------------------------------------------------------
if [ -f prisma/seed.sql ]; then
echo ""
echo "Applying reference data (prisma/seed.sql)..."
"$MYSQL_BIN" "${mysql_args[@]}" "$DB_NAME" < prisma/seed.sql
fi
# --- verify --------------------------------------------------------------
#
# Print proof rather than leaving the operator to trust us.
#
echo ""
echo '-------'
echo 'Verification'
echo '-------'
prisma migrate status
echo ""
"$MYSQL_BIN" "${mysql_args[@]}" "$DB_NAME" --table -e "
SELECT 'attachment FULLTEXT index' AS check_name,
IFNULL(MAX(INDEX_TYPE), 'MISSING') AS result
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND INDEX_NAME = 'attachment_text_index'
UNION ALL
SELECT 'tables',
CAST(COUNT(*) AS CHAR) FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME <> '_prisma_migrations'
UNION ALL
SELECT 'attachment_type seed rows',
CAST(COUNT(*) AS CHAR) FROM attachment_type
UNION ALL
-- _prisma_migrations is Prisma's own bookkeeping table and is created with
-- its default collation; only application tables are checked here.
SELECT 'non-general_ci app tables (expect 0)',
CAST(COUNT(*) AS CHAR) FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME <> '_prisma_migrations'
AND TABLE_COLLATION <> 'utf8mb4_general_ci';
"
#
# Assert that everything the application actually needs now exists.
#
# This is the check that was missing. `prisma migrate deploy` reporting success
# only means the recorded migrations ran -- it says nothing about whether the
# database ended up in the intended shape, which is precisely how production
# could pass this script and still log
# "Scraper: DISABLED - missing table scrape_cache ...".
#
# Mirrors server/helpers/Schema.js's boot verification, so a green run here
# means a green boot there.
#
FAILURES=0
require_object() {
local label="$1" count="$2"
if [ "${count:-0}" -eq 0 ]; then
echo "FAILED: missing $label" >&2
FAILURES=$((FAILURES + 1))
fi
}
require_object "table scrape_cache" "$(sql "SELECT COUNT(*) FROM information_schema.TABLES
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='scrape_cache'")"
for column in scrape_state scrape_attempts enqueued_at; do
require_object "attachment.$column" "$(sql "SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND COLUMN_NAME='$column'")"
done
require_object "attachment index attachment_scrape_state" "$(sql "SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND INDEX_NAME='attachment_scrape_state'")"
# Checked by TYPE, not just by name: a BTREE index called attachment_text_index
# would make MATCH..AGAINST return nothing at all, with no error anywhere.
FT=$(sql "SELECT IFNULL(MAX(INDEX_TYPE),'MISSING') FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA='$DB_NAME' AND INDEX_NAME='attachment_text_index'")
if [ "$FT" != "FULLTEXT" ]; then
echo "FAILED: attachment_text_index is '$FT', expected FULLTEXT." >&2
echo "Attachment search depends on MATCH..AGAINST and would silently return nothing." >&2
FAILURES=$((FAILURES + 1))
fi
if [ "$FAILURES" -gt 0 ]; then
echo "" >&2
echo "$FAILURES required object(s) missing after migrating -- the scraper will stay disabled." >&2
echo "This should not happen; check 'prisma migrate status' and the migration SQL." >&2
exit 1
fi
echo ""
echo '-------'
echo 'Done. Restart the API so it re-verifies the schema:'
echo ' pm2 restart NoteServer'
echo '-------'
package.json
+3 -4
View File
@@ -10,7 +10,7 @@
"license": "ISC",
"dependencies": {
"body-parser": "^1.19.0",
"cheerio": "^1.0.0-rc.3",
"cheerio": "^1.1.0",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"express-rate-limit": "^5.1.3",
@@ -22,8 +22,6 @@
"mysql2": "^2.2.5",
"node-tesseract-ocr": "^2.0.0",
"qrcode": "^1.4.4",
"request": "^2.88.2",
"request-promise": "^4.2.6",
"socket.io": "^2.3.0",
"speakeasy": "^2.0.0"
},
@@ -32,7 +30,8 @@
"@models": "server/models",
"@routes": "server/routes",
"@helpers": "server/helpers",
"@config": "server/config"
"@config": "server/config",
"@workers": "server/workers"
},
"devDependencies": {
"jest": "^29.7.0"
pnpm-lock.yaml
Generated
+1 -358
View File
@@ -12,7 +12,7 @@ importers:
specifier: ^1.19.0
version: 1.20.9([email protected])
cheerio:
specifier: ^1.0.0-rc.3
specifier: ^1.1.0
version: 1.2.0
dotenv:
specifier: ^8.2.0
@@ -47,12 +47,6 @@ importers:
qrcode:
specifier: ^1.4.4
version: 1.5.4
request:
specifier: ^2.88.2
version: 2.88.2
request-promise:
specifier: ^4.2.6
version: 4.2.6([email protected])
socket.io:
specifier: ^2.3.0
version: 2.5.1([email protected])
@@ -373,9 +367,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-QbJ0NTQ/I9DI3uSJA4cbexiwQeRAfjPScqIbSjUDd9TOrcg6pTkdgziesOqxBMBzit8vFCTwrP27t13vFOORRA==}
[email protected]:
resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==}
[email protected]:
resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==}
engines: {node: '>=8'}
@@ -414,22 +405,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==}
[email protected]:
resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==}
[email protected]:
resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==}
engines: {node: '>=0.8'}
[email protected]:
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
[email protected]:
resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==}
[email protected]:
resolution: {integrity: sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==}
[email protected]:
resolution: {integrity: sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@@ -477,15 +452,9 @@ packages:
engines: {node: '>=6.0.0'}
hasBin: true
[email protected]:
resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==}
[email protected]:
resolution: {integrity: sha512-gaqbzQPqOoamawKg0LGVd7SzLgXS+JH61oWprSLH+P+abTczqJbhTR8CmJ2u9/bUYNmHTGJx/UEmn6doAvvuig==}
[email protected]:
resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
[email protected]:
resolution: {integrity: sha512-FISIQxV5u91Y9gvuFq3q/tA/qTqLJn3AhRcEvyozyhJTh5OBFFtv827AdHU7Rbyct4r+Ba1/99oFa1Y3yuFsdw==}
engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@@ -545,9 +514,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-LFUqIIAwpzsIuevnKnsR9qnStBcgzPShTX61lBwN9uFkkf8o8zLiebEjbW8AYtZiF+1YKDWfdt/pWDtMuG8Xwg==}
[email protected]:
resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==}
[email protected]:
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
engines: {node: '>=10'}
@@ -591,10 +557,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
[email protected]:
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
engines: {node: '>= 0.8'}
[email protected]:
resolution: {integrity: sha512-WZveuKPeKAG9qY+FkYDeADzdHyTYdIboXS59ixDeRJL5ZhxpqUnxSOwop4FQjMsiYm3/Or8cegVbpAHNA7pHxw==}
@@ -636,9 +598,6 @@ packages:
resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==}
engines: {node: '>= 0.6'}
[email protected]:
resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==}
[email protected]:
resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
@@ -658,10 +617,6 @@ packages:
resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==}
engines: {node: '>= 6'}
[email protected]:
resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==}
engines: {node: '>=0.10'}
[email protected]:
resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
peerDependencies:
@@ -720,10 +675,6 @@ packages:
resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==}
engines: {node: '>=0.10.0'}
[email protected]:
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
[email protected]:
resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
engines: {node: '>=0.10'}
@@ -772,9 +723,6 @@ packages:
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
[email protected]:
resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==}
[email protected]:
resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==}
@@ -874,16 +822,6 @@ packages:
resolution: {integrity: sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw==}
engines: {node: '>= 0.10.0'}
[email protected]:
resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
[email protected]:
resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==}
engines: {'0': node >=0.6.0}
[email protected]:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
[email protected]:
resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}
@@ -902,13 +840,6 @@ packages:
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
engines: {node: '>=8'}
[email protected]:
resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==}
[email protected]:
resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==}
engines: {node: '>= 0.12'}
[email protected]:
resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
engines: {node: '>= 0.6'}
@@ -955,9 +886,6 @@ packages:
resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==}
engines: {node: '>=10'}
[email protected]:
resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==}
[email protected]:
resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]
@@ -974,15 +902,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
[email protected]:
resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==}
engines: {node: '>=4'}
[email protected]:
resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==}
engines: {node: '>=6'}
deprecated: this library is no longer supported
[email protected]:
resolution: {integrity: sha512-G1LWKhDSvhGeAQ8mPVQlqNcOB2sJdwATtZKl2pDKKHfpf/rYj24lkinxf69blJbnsvtqqNU+L3SL50vzZhXOnw==}
@@ -1015,10 +934,6 @@ packages:
resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==}
engines: {node: '>= 0.8'}
[email protected]:
resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==}
engines: {node: '>=0.8', npm: '>=1.3.7'}
[email protected]:
resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==}
engines: {node: '>=10.17.0'}
@@ -1080,9 +995,6 @@ packages:
resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
engines: {node: '>=8'}
[email protected]:
resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==}
[email protected]:
resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==}
@@ -1095,9 +1007,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
[email protected]:
resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==}
[email protected]:
resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==}
engines: {node: '>=8'}
@@ -1258,9 +1167,6 @@ packages:
resolution: {integrity: sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==}
hasBin: true
[email protected]:
resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==}
[email protected]:
resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==}
engines: {node: '>=6'}
@@ -1269,15 +1175,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
[email protected]:
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
[email protected]:
resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==}
[email protected]:
resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
[email protected]:
resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
engines: {node: '>=6'}
@@ -1287,10 +1184,6 @@ packages:
resolution: {integrity: sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==}
engines: {node: '>=4', npm: '>=1.4.28'}
[email protected]:
resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==}
engines: {node: '>=0.6.0'}
[email protected]:
resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==}
@@ -1333,9 +1226,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==}
[email protected]:
resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==}
[email protected]:
resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==}
@@ -1457,9 +1347,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
[email protected]:
resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==}
[email protected]:
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
engines: {node: '>=0.10.0'}
@@ -1536,9 +1423,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==}
[email protected]:
resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==}
[email protected]:
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
@@ -1573,13 +1457,6 @@ packages:
resolution: {integrity: sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==}
engines: {node: '>= 0.10'}
[email protected]:
resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==}
[email protected]:
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
[email protected]:
resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==}
@@ -1592,10 +1469,6 @@ packages:
resolution: {integrity: sha512-5Px6J7rfHBAGUhNQ3es4YzcOdBBw0SDcVtThJml4RdnKw4tKs2epA0kPZ+K8X+WiHktT93RHVcQ/3ym5aTCgtg==}
engines: {node: '>=0.6'}
[email protected]:
resolution: {integrity: sha512-mzR4sElr1bfCaPJe7m8ilJ6ZXdDaGoObcYR0ZHSsktM/Lt21MVHj5De30GQH2eiZ1qGRTO7LCAzQsUeXTNexWQ==}
engines: {node: '>=0.6'}
[email protected]:
resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
engines: {node: '>= 0.6'}
@@ -1613,24 +1486,6 @@ packages:
[email protected]:
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
[email protected]:
resolution: {integrity: sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==}
engines: {node: '>=0.10.0'}
peerDependencies:
request: ^2.34
[email protected]:
resolution: {integrity: sha512-HCHI3DJJUakkOr8fNoCc73E5nU5bqITjOYFMDrKHYOXWXrgD/SBaC7LjwuPymUprRyuF06UK7hd/lMHkmUXglQ==}
engines: {node: '>=0.10.0'}
deprecated: request-promise has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142
peerDependencies:
request: ^2.34
[email protected]:
resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==}
engines: {node: '>= 6'}
deprecated: request has been deprecated, see https://github.com/request/request/issues/3142
[email protected]:
resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
engines: {node: '>=0.10.0'}
@@ -1762,11 +1617,6 @@ packages:
resolution: {integrity: sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==}
engines: {node: '>= 0.6'}
[email protected]:
resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==}
engines: {node: '>=0.10.0'}
hasBin: true
[email protected]:
resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==}
engines: {node: '>=10'}
@@ -1775,10 +1625,6 @@ packages:
resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==}
engines: {node: '>= 0.8'}
[email protected]:
resolution: {integrity: sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g==}
engines: {node: '>=0.10.0'}
[email protected]:
resolution: {integrity: sha512-jos8u++JKm0ARcSUTAZXOVC0mSox7Bhn6sBgty73P1f3JGf7yG2clTbBNHUdde/kdvP2FESam+vM6l8jBrNxHA==}
engines: {node: '>=0.8.0'}
@@ -1843,16 +1689,6 @@ packages:
resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
engines: {node: '>=0.6'}
[email protected]:
resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==}
engines: {node: '>=0.8'}
[email protected]:
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
[email protected]:
resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==}
[email protected]:
resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==}
engines: {node: '>=4'}
@@ -1885,9 +1721,6 @@ packages:
peerDependencies:
browserslist: '>= 4.21.0'
[email protected]:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
[email protected]:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
@@ -1895,11 +1728,6 @@ packages:
resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}
engines: {node: '>= 0.4.0'}
[email protected]:
resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==}
deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).
hasBin: true
[email protected]:
resolution: {integrity: sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==}
engines: {node: '>=10.12.0'}
@@ -1908,10 +1736,6 @@ packages:
resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}
engines: {node: '>= 0.8'}
[email protected]:
resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==}
engines: {'0': node >=0.6.0}
[email protected]:
resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==}
@@ -2449,13 +2273,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
fast-deep-equal: 3.1.3
fast-json-stable-stringify: 2.1.0
json-schema-traverse: 0.4.1
uri-js: 4.4.1
[email protected]:
dependencies:
type-fest: 0.21.3
@@ -2487,18 +2304,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
safer-buffer: 2.1.2
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected](@babel/[email protected]([email protected]))([email protected]):
dependencies:
'@babel/core': 7.29.7([email protected])
@@ -2566,14 +2371,8 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
tweetnacl: 0.14.5
[email protected]: {}
[email protected]: {}
[email protected]([email protected]):
dependencies:
bytes: 3.1.2
@@ -2643,8 +2442,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]:
dependencies:
ansi-styles: 4.3.0
@@ -2701,10 +2498,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
delayed-stream: 1.0.0
[email protected]: {}
[email protected]: {}
@@ -2736,8 +2529,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected](@types/[email protected])([email protected]):
@@ -2771,10 +2562,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
assert-plus: 1.0.0
[email protected]([email protected]):
dependencies:
ms: 2.0.0
@@ -2811,8 +2598,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -2856,11 +2641,6 @@ snapshots:
es-errors: 1.3.0
gopd: 1.2.0
[email protected]:
dependencies:
jsbn: 0.1.1
safer-buffer: 2.1.2
[email protected]:
dependencies:
safe-buffer: 5.2.1
@@ -3007,12 +2787,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3040,14 +2814,6 @@ snapshots:
locate-path: 5.0.0
path-exists: 4.0.0
[email protected]: {}
[email protected]:
dependencies:
asynckit: 0.4.0
combined-stream: 1.0.8
mime-types: 2.1.35
[email protected]: {}
[email protected]: {}
@@ -3089,10 +2855,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
assert-plus: 1.0.0
[email protected]:
dependencies:
fs.realpath: 1.0.0
@@ -3115,13 +2877,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]:
dependencies:
ajv: 6.15.0
har-schema: 2.0.0
[email protected]:
dependencies:
isarray: 2.0.1
@@ -3155,12 +2910,6 @@ snapshots:
statuses: 2.0.2
toidentifier: 1.0.1
[email protected]:
dependencies:
assert-plus: 1.0.0
jsprim: 1.4.2
sshpk: 1.18.0
[email protected]: {}
[email protected]:
@@ -3205,8 +2954,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -3215,8 +2962,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]([email protected]):
@@ -3573,18 +3318,10 @@ snapshots:
argparse: 1.0.10
esprima: 4.0.1
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3600,13 +3337,6 @@ snapshots:
ms: 2.1.3
semver: 5.7.2
[email protected]:
dependencies:
assert-plus: 1.0.0
extsprintf: 1.3.0
json-schema: 0.4.0
verror: 1.10.0
[email protected]:
dependencies:
buffer-equal-constant-time: 1.0.1
@@ -3642,8 +3372,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3751,8 +3479,6 @@ snapshots:
dependencies:
boolbase: 1.0.0
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -3819,8 +3545,6 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -3851,12 +3575,6 @@ snapshots:
forwarded: 0.2.0
ipaddr.js: 1.9.1
[email protected]:
dependencies:
punycode: 2.3.1
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3870,8 +3588,6 @@ snapshots:
es-define-property: 1.0.1
side-channel: 1.1.1
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -3900,42 +3616,6 @@ snapshots:
string_decoder: 1.1.1
util-deprecate: 1.0.2
[email protected]([email protected]):
dependencies:
lodash: 4.18.1
request: 2.88.2
[email protected]([email protected]):
dependencies:
bluebird: 3.7.2
request: 2.88.2
request-promise-core: 1.1.4([email protected])
stealthy-require: 1.1.1
tough-cookie: 2.5.0
[email protected]:
dependencies:
aws-sign2: 0.7.0
aws4: 1.13.2
caseless: 0.12.0
combined-stream: 1.0.8
extend: 3.0.2
forever-agent: 0.6.1
form-data: 2.3.3
har-validator: 5.1.5
http-signature: 1.2.0
is-typedarray: 1.0.0
isstream: 0.1.2
json-stringify-safe: 5.0.1
mime-types: 2.1.35
oauth-sign: 0.9.0
performance-now: 2.1.0
qs: 6.5.5
safe-buffer: 5.2.1
tough-cookie: 2.5.0
tunnel-agent: 0.6.0
uuid: 3.4.0
[email protected]: {}
[email protected]: {}
@@ -4104,26 +3784,12 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
asn1: 0.2.6
assert-plus: 1.0.0
bcrypt-pbkdf: 1.0.2
dashdash: 1.14.1
ecc-jsbn: 0.1.2
getpass: 0.1.7
jsbn: 0.1.1
safer-buffer: 2.1.2
tweetnacl: 0.14.5
[email protected]:
dependencies:
escape-string-regexp: 2.0.0
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
@@ -4179,17 +3845,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
psl: 1.15.0
punycode: 2.3.1
[email protected]:
dependencies:
safe-buffer: 5.2.1
[email protected]: {}
[email protected]: {}
[email protected]: {}
@@ -4213,16 +3868,10 @@ snapshots:
escalade: 3.2.0
picocolors: 1.1.1
[email protected]:
dependencies:
punycode: 2.3.1
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]:
dependencies:
'@jridgewell/trace-mapping': 0.3.31
@@ -4231,12 +3880,6 @@ snapshots:
[email protected]: {}
[email protected]:
dependencies:
assert-plus: 1.0.0
core-util-is: 1.0.2
extsprintf: 1.3.0
[email protected]:
dependencies:
makeerror: 1.0.12
prisma.config.mjs
+32
View File
@@ -0,0 +1,32 @@
//
// Prisma CLI configuration.
//
// Prisma 7 removed `url` from the datasource block in schema.prisma; the
// connection string for migrations lives here instead.
//
// This file configures the CLI only. The running server does not use Prisma --
// it talks to MariaDB through server/config/database.js (mysql2) with raw SQL.
// Prisma is a development and admin tool here, for schema migrations and
// scaffolding.
//
// DATABASE_URL must point at an account with DDL rights. In dev that is the
// root user exported by shell.nix; in production migrateDatabase.sh composes it
// from the admin credentials in ~/.env. It must NEVER be the least-privilege
// `app` account the server runs as, which deliberately holds no CREATE/ALTER.
//
// .mjs rather than .js because this project is CommonJS and Prisma's example
// config is ESM; the explicit extension avoids any ambiguity.
//
export default {
schema: 'prisma/schema.prisma',
migrations: {
path: 'prisma/migrations',
},
datasource: {
// process.env directly rather than env() -- the helper throws when the
// variable is unset, which makes the failure harder to read.
url: process.env.DATABASE_URL,
},
}
prisma/migrations/0002_scrape_objects_repair/migration.sql
+74
View File
@@ -0,0 +1,74 @@
--
-- Repair: converge environments that were baselined before the scrape objects
-- existed.
--
-- WHY THIS EXISTS
--
-- The 0_init baseline was generated from a database that already had
-- scrape_cache and the attachment.scrape_* columns, on the understanding that
-- every environment was in that state. Production was not. Because
-- `prisma migrate resolve --applied 0_init` records a migration WITHOUT
-- executing it, production ended up with 0_init marked applied while those
-- objects were never created -- so the scraper logged:
--
-- Scraper: DISABLED - missing table scrape_cache, attachment.scrape_state, ...
--
-- Re-running 0_init is not an option (it would try to CREATE TABLE over ~2200
-- rows of live data), so this additive, idempotent migration brings any
-- environment up to the intended schema instead.
--
-- Every statement uses IF NOT EXISTS, so this is a clean no-op on environments
-- that already had these objects. MariaDB 11.4 supports that syntax for columns
-- and indexes; both dev and production are MariaDB, so this is safe here. It
-- would NOT be valid on MySQL 8.
--
-- The definitions below are identical to those in 0_init -- deliberately so,
-- because the end state must be the same whichever path an environment took.
--
-- One row per scraped URL, shared across users and notes.
--
-- url_hash carries the UNIQUE index rather than `url` because a UNIQUE on
-- varchar(1000) utf8mb4 is 4000 bytes and exceeds InnoDB's 3072-byte key limit.
--
-- Collation is explicit: Prisma renders utf8mb4_unicode_ci, but every table in
-- this database is utf8mb4_general_ci, and `prisma migrate diff` does not
-- report the difference.
CREATE TABLE IF NOT EXISTS `scrape_cache` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`url_hash` CHAR(64) NOT NULL,
`url` VARCHAR(1000) NOT NULL DEFAULT '',
`canonical_url` VARCHAR(1000) NULL,
`status` SMALLINT NOT NULL DEFAULT 0,
`title` VARCHAR(512) NULL,
`site_name` VARCHAR(255) NULL,
`description` MEDIUMTEXT NULL,
`keywords` VARCHAR(512) NULL,
`search_text` MEDIUMTEXT NULL,
`image_url` VARCHAR(1000) NULL,
`file_location` VARCHAR(500) NULL,
`etag` VARCHAR(255) NULL,
`last_modified` VARCHAR(128) NULL,
`content_type` VARCHAR(128) NULL,
`fetch_state` VARCHAR(20) NOT NULL DEFAULT 'pending',
`failure_count` INTEGER NOT NULL DEFAULT 0,
`last_error` VARCHAR(255) NULL,
`fetched_at` INTEGER NULL,
`expires_at` INTEGER NULL,
UNIQUE INDEX `scrape_cache_url_hash`(`url_hash`),
INDEX `scrape_cache_expires_at`(`expires_at`),
INDEX `scrape_cache_fetch_state`(`fetch_state`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- Per-attachment scrape lifecycle, so the in-memory queue survives a restart.
--
-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
-- DEFAULT 2 is load-bearing -- it keeps every pre-existing attachment row
-- (~2200 in production) out of the scraper's boot recovery sweep.
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_state` TINYINT NOT NULL DEFAULT 2;
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_attempts` TINYINT NOT NULL DEFAULT 0;
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `enqueued_at` INTEGER NULL;
CREATE INDEX IF NOT EXISTS `attachment_scrape_state` ON `attachment` (`scrape_state`, `enqueued_at`);
prisma/migrations/0_init/migration.sql
+211
View File
@@ -0,0 +1,211 @@
--
-- Baseline migration: the schema as it existed when Prisma was adopted.
--
-- Generated with:
-- prisma migrate diff --from-empty --to-schema prisma/schema.prisma --script
--
-- HAND-CORRECTED, and it matters: Prisma renders `COLLATE utf8mb4_unicode_ci`
-- for MySQL, but every live table is `utf8mb4_general_ci`. Prisma does not
-- model collation and its own `migrate diff` does NOT report the difference, so
-- without this correction a database rebuilt from empty would silently differ
-- from production in string comparison, sorting and FULLTEXT ranking.
-- If you regenerate this file, re-apply the collation fix. There is a test for
-- it in server/tests/schema/migrations.test.js.
--
-- Things Prisma cannot express, which therefore need handling elsewhere:
-- * the DATABASE-level default collation -- set by migrateDatabase.sh when it
-- creates the database, since tables created later without an explicit
-- COLLATE would otherwise inherit the server default
-- * ENGINE=InnoDB (relies on the server default, which is InnoDB)
-- * AUTO_INCREMENT start values (cosmetic; a fresh database starting at 1 is
-- correct)
-- * the COMMENT on note_raw_text.text
-- * row data -- see prisma/seed.sql for the attachment_type lookup rows
--
-- Context carried over from the migration this replaces: scrape_cache.url_hash
-- carries the UNIQUE index rather than `url` because a UNIQUE on
-- varchar(1000) utf8mb4 is 4000 bytes and exceeds InnoDB's 3072-byte key limit;
-- and attachment.scrape_state DEFAULT 2 ("done") is load-bearing -- it keeps
-- pre-existing attachment rows out of the scraper's boot recovery sweep.
--
-- On environments that already had this schema (dev and production), this file
-- is NOT executed -- it is marked applied with `prisma migrate resolve`.
--
-- CreateTable
CREATE TABLE `attachment` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`note_id` INTEGER UNSIGNED NULL,
`user_id` INTEGER UNSIGNED NOT NULL,
`attachment_type` INTEGER UNSIGNED NULL,
`text` MEDIUMTEXT NULL,
`url` VARCHAR(1000) NULL DEFAULT '',
`last_indexed` INTEGER NULL,
`file_location` VARCHAR(500) NULL,
`visible` TINYINT NULL DEFAULT 1,
`scrape_state` TINYINT NOT NULL DEFAULT 2,
`scrape_attempts` TINYINT NOT NULL DEFAULT 0,
`enqueued_at` INTEGER NULL,
INDEX `attachment_scrape_state`(`scrape_state`, `enqueued_at`),
INDEX `attachment_type`(`attachment_type`),
INDEX `note_id`(`note_id`),
INDEX `user_id`(`user_id`),
FULLTEXT INDEX `attachment_text_index`(`text`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `attachment_type` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`mime_type` VARCHAR(500) NULL,
`extension` VARCHAR(10) NULL,
`name` VARCHAR(50) NULL,
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `note` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`user_id` INTEGER UNSIGNED NULL,
`note_raw_text_id` INTEGER NULL,
`opened` INTEGER NULL,
`created` INTEGER NULL,
`pinned` INTEGER NULL DEFAULT 0,
`archived` INTEGER NULL DEFAULT 0,
`trashed` TINYINT NULL DEFAULT 0,
`color` VARCHAR(500) NULL,
`shared` INTEGER NULL DEFAULT 0,
`share_user_id` INTEGER NULL,
`quick_note` INTEGER NULL DEFAULT 0,
`indexed` TINYINT NULL DEFAULT 0,
`snippet` VARCHAR(1300) NULL,
`snippet_salt` VARCHAR(1300) NULL,
`encrypted_share_password_key` VARCHAR(1300) NULL,
INDEX `user_note`(`user_id`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `note_raw_text` (
`id` INTEGER NOT NULL AUTO_INCREMENT,
`text` LONGTEXT NULL,
`updated` BIGINT NULL,
`salt` VARCHAR(1300) NULL,
`last_decrypted_date` INTEGER NULL,
`decrypt_attempts_count` INTEGER NULL DEFAULT 0,
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `note_tag` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`note_id` INTEGER UNSIGNED NULL,
`tag_id` INTEGER UNSIGNED NULL,
`user_id` INTEGER UNSIGNED NULL,
INDEX `note_id`(`note_id`),
INDEX `tag_id`(`tag_id`),
INDEX `user_id`(`user_id`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `scrape_cache` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`url_hash` CHAR(64) NOT NULL,
`url` VARCHAR(1000) NOT NULL DEFAULT '',
`canonical_url` VARCHAR(1000) NULL,
`status` SMALLINT NOT NULL DEFAULT 0,
`title` VARCHAR(512) NULL,
`site_name` VARCHAR(255) NULL,
`description` MEDIUMTEXT NULL,
`keywords` VARCHAR(512) NULL,
`search_text` MEDIUMTEXT NULL,
`image_url` VARCHAR(1000) NULL,
`file_location` VARCHAR(500) NULL,
`etag` VARCHAR(255) NULL,
`last_modified` VARCHAR(128) NULL,
`content_type` VARCHAR(128) NULL,
`fetch_state` VARCHAR(20) NOT NULL DEFAULT 'pending',
`failure_count` INTEGER NOT NULL DEFAULT 0,
`last_error` VARCHAR(255) NULL,
`fetched_at` INTEGER NULL,
`expires_at` INTEGER NULL,
UNIQUE INDEX `scrape_cache_url_hash`(`url_hash`),
INDEX `scrape_cache_expires_at`(`expires_at`),
INDEX `scrape_cache_fetch_state`(`fetch_state`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `tag` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`text` VARCHAR(400) NULL,
`hash` INTEGER NULL,
INDEX `hash`(`hash`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `user` (
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`username` VARCHAR(150) NOT NULL DEFAULT '',
`password` TEXT NOT NULL,
`salt` VARCHAR(500) NOT NULL DEFAULT '',
`iterations` INTEGER NOT NULL,
`last_login` DATETIME(0) NULL,
`created` DATETIME(0) NULL,
`two_fa_secret` VARCHAR(50) NULL,
`two_fa_enabled` TINYINT NULL DEFAULT 0,
`two_fa_every_login` TINYINT NULL DEFAULT 0,
`pushkey` VARCHAR(100) NULL,
UNIQUE INDEX `id_UNIQUE`(`id`),
UNIQUE INDEX `username_UNIQUE`(`username`),
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `user_active_session` (
`id` INTEGER NOT NULL AUTO_INCREMENT,
`salt` VARCHAR(100) NULL,
`encrypted_master_password` VARCHAR(500) NULL,
`created` INTEGER NULL,
`uses` INTEGER NULL,
`user_hash` VARCHAR(500) NULL,
`active` TINYINT NULL DEFAULT 1,
`session_id` VARCHAR(100) NULL,
`last_used` INTEGER NULL,
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `user_encrypted_search_index` (
`id` INTEGER NOT NULL AUTO_INCREMENT,
`user_id` INTEGER NULL,
`salt` VARCHAR(100) NULL,
`index` LONGTEXT NULL,
`last_update` INTEGER NULL,
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
-- CreateTable
CREATE TABLE `user_key` (
`id` INTEGER NOT NULL AUTO_INCREMENT,
`user_id` INTEGER NULL,
`salt` VARCHAR(1300) NULL,
`key` VARCHAR(5000) NULL,
`created` INTEGER NULL,
`public_key` VARCHAR(5000) NULL,
`private_key_encrypted` VARCHAR(5000) NULL,
PRIMARY KEY (`id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
prisma/migrations/migration_lock.toml
+3
View File
@@ -0,0 +1,3 @@
# Please do not edit this file manually
# It should be added in your version-control system (e.g., Git)
provider = "mysql"
prisma/schema.prisma
+151
View File
@@ -0,0 +1,151 @@
datasource db {
provider = "mysql"
}
model attachment {
id Int @id @default(autoincrement()) @db.UnsignedInt
note_id Int? @db.UnsignedInt
user_id Int @db.UnsignedInt
attachment_type Int? @db.UnsignedInt
text String? @db.MediumText
url String? @default("") @db.VarChar(1000)
last_indexed Int?
file_location String? @db.VarChar(500)
visible Int? @default(1) @db.TinyInt
scrape_state Int @default(2) @db.TinyInt
scrape_attempts Int @default(0) @db.TinyInt
enqueued_at Int?
@@index([scrape_state, enqueued_at], map: "attachment_scrape_state")
@@index([attachment_type], map: "attachment_type")
@@index([note_id], map: "note_id")
@@index([user_id], map: "user_id")
@@fulltext([text], map: "attachment_text_index")
}
model attachment_type {
id Int @id @default(autoincrement()) @db.UnsignedInt
mime_type String? @db.VarChar(500)
extension String? @db.VarChar(10)
name String? @db.VarChar(50)
}
model note {
id Int @id @default(autoincrement()) @db.UnsignedInt
user_id Int? @db.UnsignedInt
note_raw_text_id Int?
opened Int?
created Int?
pinned Int? @default(0)
archived Int? @default(0)
trashed Int? @default(0) @db.TinyInt
color String? @db.VarChar(500)
shared Int? @default(0)
share_user_id Int?
quick_note Int? @default(0)
indexed Int? @default(0) @db.TinyInt
snippet String? @db.VarChar(1300)
snippet_salt String? @db.VarChar(1300)
encrypted_share_password_key String? @db.VarChar(1300)
@@index([user_id], map: "user_note")
}
/// This model or at least one of its fields has comments in the database, and requires an additional setup for migrations: Read more: https://pris.ly/d/database-comments
model note_raw_text {
id Int @id @default(autoincrement())
text String? @db.LongText
updated BigInt?
salt String? @db.VarChar(1300)
last_decrypted_date Int?
decrypt_attempts_count Int? @default(0)
}
model note_tag {
id Int @id @default(autoincrement()) @db.UnsignedInt
note_id Int? @db.UnsignedInt
tag_id Int? @db.UnsignedInt
user_id Int? @db.UnsignedInt
@@index([note_id], map: "note_id")
@@index([tag_id], map: "tag_id")
@@index([user_id], map: "user_id")
}
model scrape_cache {
id Int @id @default(autoincrement()) @db.UnsignedInt
url_hash String @unique(map: "scrape_cache_url_hash") @db.Char(64)
url String @default("") @db.VarChar(1000)
canonical_url String? @db.VarChar(1000)
status Int @default(0) @db.SmallInt
title String? @db.VarChar(512)
site_name String? @db.VarChar(255)
description String? @db.MediumText
keywords String? @db.VarChar(512)
search_text String? @db.MediumText
image_url String? @db.VarChar(1000)
file_location String? @db.VarChar(500)
etag String? @db.VarChar(255)
last_modified String? @db.VarChar(128)
content_type String? @db.VarChar(128)
fetch_state String @default("pending") @db.VarChar(20)
failure_count Int @default(0)
last_error String? @db.VarChar(255)
fetched_at Int?
expires_at Int?
@@index([expires_at], map: "scrape_cache_expires_at")
@@index([fetch_state], map: "scrape_cache_fetch_state")
}
model tag {
id Int @id @default(autoincrement()) @db.UnsignedInt
text String? @db.VarChar(400)
hash Int?
@@index([hash], map: "hash")
}
model user {
id Int @id @unique(map: "id_UNIQUE") @default(autoincrement()) @db.UnsignedInt
username String @unique(map: "username_UNIQUE") @default("") @db.VarChar(150)
password String @db.Text
salt String @default("") @db.VarChar(500)
iterations Int
last_login DateTime? @db.DateTime(0)
created DateTime? @db.DateTime(0)
two_fa_secret String? @db.VarChar(50)
two_fa_enabled Int? @default(0) @db.TinyInt
two_fa_every_login Int? @default(0) @db.TinyInt
pushkey String? @db.VarChar(100)
}
model user_active_session {
id Int @id @default(autoincrement())
salt String? @db.VarChar(100)
encrypted_master_password String? @db.VarChar(500)
created Int?
uses Int?
user_hash String? @db.VarChar(500)
active Int? @default(1) @db.TinyInt
session_id String? @db.VarChar(100)
last_used Int?
}
model user_encrypted_search_index {
id Int @id @default(autoincrement())
user_id Int?
salt String? @db.VarChar(100)
index String? @db.LongText
last_update Int?
}
model user_key {
id Int @id @default(autoincrement())
user_id Int?
salt String? @db.VarChar(1300)
key String? @db.VarChar(5000)
created Int?
public_key String? @db.VarChar(5000)
private_key_encrypted String? @db.VarChar(5000)
}
prisma/seed.sql
+24
View File
@@ -0,0 +1,24 @@
--
-- Reference data required for a working database.
--
-- Applied by ./migrateDatabase.sh after migrations. Kept out of the migration
-- history on purpose: environments that were baselined with
-- `prisma migrate resolve` never execute 0_init, so seed rows placed inside it
-- would silently never exist there.
--
-- Idempotent, so re-running is harmless.
--
-- attachment_type is a lookup table the application joins against.
-- attachment_type = 1 is hardcoded for scraped URLs, 2 for uploaded images,
-- 3 for PDFs (see server/models/Attachment.js). Without these rows a freshly
-- scaffolded database creates attachments that reference nothing.
INSERT INTO `attachment_type` (`id`, `mime_type`, `extension`, `name`) VALUES
(1, NULL, NULL, 'URL'),
(2, NULL, '.jpg', 'Image - Jpg'),
(3, NULL, '.pdf', 'PDF'),
(4, NULL, '.doc', 'DOC')
ON DUPLICATE KEY UPDATE
`mime_type` = VALUES(`mime_type`),
`extension` = VALUES(`extension`),
`name` = VALUES(`name`);
server/config/scrape.js
+70
View File
@@ -0,0 +1,70 @@
//
// All scraper tunables in one place, so nothing downstream is a magic number.
//
// Env overrides exist for the two kill switches only. Everything else is a
// constant on purpose: the politeness numbers are the difference between
// looking like a browser and looking like a bot, and they should not drift
// per-environment without a code change.
//
const isTest = process.env.NODE_ENV === 'test'
const config = {
// Kill switches
enabled: process.env.SCRAPE_ENABLED !== '0',
// The background worker is off under Jest so tests drive the queue
// deterministically via ScrapeQueue.drain() instead of racing it.
workerEnabled: process.env.SCRAPE_WORKER ? process.env.SCRAPE_WORKER === '1' : !isTest,
// Timeouts
pageTimeoutMs: 8000, // single HTTP request
totalBudgetMs: 25000, // all hops + retries for one URL
imageTimeoutMs: 10000,
// Size caps. Content-Length is optional and can lie, so these are also
// enforced while streaming the body.
maxHtmlBytes: 2 * 1024 * 1024,
maxImageBytes: 5 * 1024 * 1024,
maxRobotsBytes: 100 * 1024,
// Redirects
maxRedirects: 5,
// Retries. Only for transient classes - a 404 is an answer, not a failure.
retries: 2,
backoffBaseMs: 500,
backoffMaxMs: 8000,
maxRetryAfterMs: 60 * 1000,
// Politeness. One request at a time per host, spaced out, is what actually
// stops us looking like a crawler.
maxConcurrentFetches: 4,
perHostConcurrency: 1,
perHostMinSpacingMs: 1000,
perHostMaxSpacingMs: 3000,
maxCrawlDelayMs: 30 * 1000,
// Backpressure
maxQueueLength: 500,
maxPendingPerUser: 50,
// Identity. Stable salt, so a host always sees the same browser across
// restarts. Changing this re-rolls every host's assigned profile.
profileSalt: 'ss-scrape-v1',
robotsUaToken: 'SolidScribeBot',
robotsTtlMs: 24 * 60 * 60 * 1000,
robotsNegativeTtlMs: 60 * 60 * 1000,
robotsErrorTtlMs: 10 * 60 * 1000,
robotsMaxOrigins: 2000,
// Parsing / storage
maxSearchTextBytes: 8000,
maxKeywordSourceChars: 6000,
maxUrlLength: 1000, // matches attachment.url varchar(1000)
// staticFiles is served from our own origin via /api/static, so an SVG
// with an inline script would be same-origin. Off by default.
allowSvgThumbnails: false,
}
module.exports = config
server/ecosystem.config.js
+4
View File
@@ -2,6 +2,10 @@ module.exports = {
apps : [{
name: 'NoteServer',
script: 'index.js',
// Both relative to this file, not to whatever directory pm2 was invoked
// from. deploy.sh runs pm2 from the repo root, so without cwd pm2 resolves
// script against the root and dies with "Script not found: .../index.js".
cwd: __dirname,
// Options reference: https://pm2.io/doc/en/runtime/reference/ecosystem-file/
// args: 'one two',
server/helpers/BrowserProfiles.js
+195
View File
@@ -0,0 +1,195 @@
//
// Browser identities for outbound fetches.
//
// The old code sent a single hardcoded Googlebot User-Agent. That was removed
// deliberately: Googlebot is verifiable by reverse DNS, so the claim is trivially
// disproven, and sites that check it block the source IP outright. Some also
// serve crawlers a stripped-down page.
//
// What replaces it is an honest, *coherent* browser identity. Coherence is the
// point - a Chrome User-Agent shipping Firefox's Accept header, or sec-ch-ua on
// a Safari request, is a stronger bot signal than any UA string. So each profile
// carries the complete header set its family really sends, and nothing mixes.
//
// Profiles are assigned stably per host, not randomly per request: a site should
// see one returning browser, not a rotating cast.
//
const config = require('@config/scrape')
let BrowserProfiles = module.exports = {}
const CHROME_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'
const FIREFOX_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
const SAFARI_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
const PROFILES = BrowserProfiles.PROFILES = [
{
id: 'chrome-win',
family: 'chrome',
ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
accept: CHROME_ACCEPT,
acceptLanguage: 'en-US,en;q=0.9',
secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
secChUaMobile: '?0',
secChUaPlatform: '"Windows"',
},
{
id: 'chrome-mac',
family: 'chrome',
ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
accept: CHROME_ACCEPT,
acceptLanguage: 'en-US,en;q=0.9',
secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
secChUaMobile: '?0',
secChUaPlatform: '"macOS"',
},
{
id: 'firefox-win',
family: 'firefox',
ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
accept: FIREFOX_ACCEPT,
acceptLanguage: 'en-US,en;q=0.5',
},
{
id: 'firefox-linux',
family: 'firefox',
ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0',
accept: FIREFOX_ACCEPT,
acceptLanguage: 'en-US,en;q=0.5',
},
{
id: 'safari-mac',
family: 'safari',
ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
accept: SAFARI_ACCEPT,
acceptLanguage: 'en-US,en;q=0.9',
},
]
// FNV-1a. Not cryptographic - we only need a stable, well-spread bucket.
const fnv1a = (input) => {
let hash = 0x811c9dc5
for (let i = 0; i < input.length; i++) {
hash ^= input.charCodeAt(i)
hash = Math.imul(hash, 0x01000193) >>> 0
}
return hash >>> 0
}
BrowserProfiles.__fnv1a = fnv1a
//
// Same host always gets the same profile, across requests and across restarts
// (the salt is a constant in config, not a per-boot random).
//
BrowserProfiles.profileForHost = (hostname) => {
const key = require('@helpers/UrlSafety').registrableHost(hostname) + config.profileSalt
return PROFILES[fnv1a(key) % PROFILES.length]
}
BrowserProfiles.profileById = (id) => PROFILES.find(profile => profile.id === id) || PROFILES[0]
//
// Sec-Fetch-Site is relative: "none" for a URL typed into the bar, "same-origin"
// when the referrer shares the origin, "cross-site" otherwise. Getting this
// wrong while sending a Referer is exactly the kind of incoherence that flags.
//
const fetchSite = (targetUrl, referer) => {
if (!referer) { return 'none' }
try {
const target = new URL(targetUrl)
const from = new URL(referer)
if (target.origin === from.origin) { return 'same-origin' }
if (UrlSafetyHost(target) === UrlSafetyHost(from)) { return 'same-site' }
return 'cross-site'
} catch (error) {
return 'none'
}
}
const UrlSafetyHost = (url) => require('@helpers/UrlSafety').registrableHost(url.hostname)
const applyChromiumHints = (headers, profile) => {
if (profile.family !== 'chrome') { return headers }
headers['sec-ch-ua'] = profile.secChUa
headers['sec-ch-ua-mobile'] = profile.secChUaMobile
headers['sec-ch-ua-platform'] = profile.secChUaPlatform
return headers
}
//
// Headers for a top-level document navigation.
//
BrowserProfiles.documentHeaders = (profile, { url, referer } = {}) => {
const headers = {
'User-Agent': profile.ua,
'Accept': profile.accept,
'Accept-Language': profile.acceptLanguage,
'Accept-Encoding': 'gzip, deflate, br',
}
// Safari sends neither sec-ch-ua nor Sec-Fetch-*; Firefox sends Sec-Fetch-*
// but no client hints; Chromium sends both.
if (profile.family !== 'safari') {
headers['Sec-Fetch-Dest'] = 'document'
headers['Sec-Fetch-Mode'] = 'navigate'
headers['Sec-Fetch-Site'] = fetchSite(url, referer)
headers['Sec-Fetch-User'] = '?1'
headers['Upgrade-Insecure-Requests'] = '1'
}
applyChromiumHints(headers, profile)
if (referer) { headers['Referer'] = referer }
return headers
}
//
// Headers for a subresource image load, as issued from the page it was found on.
//
BrowserProfiles.imageHeaders = (profile, { url, pageUrl } = {}) => {
const headers = {
'User-Agent': profile.ua,
'Accept': profile.family === 'chrome'
? 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8'
: 'image/avif,image/webp,*/*',
'Accept-Language': profile.acceptLanguage,
'Accept-Encoding': 'gzip, deflate, br',
}
if (profile.family !== 'safari') {
headers['Sec-Fetch-Dest'] = 'image'
headers['Sec-Fetch-Mode'] = 'no-cors'
headers['Sec-Fetch-Site'] = fetchSite(url, pageUrl)
}
applyChromiumHints(headers, profile)
if (pageUrl) { headers['Referer'] = pageUrl }
return headers
}
//
// Headers for robots.txt. Same identity as the page fetch, but it is not a
// navigation - a browser would never send Sec-Fetch-Dest: document for this.
//
BrowserProfiles.robotsHeaders = (profile) => {
const headers = {
'User-Agent': profile.ua,
'Accept': 'text/plain,*/*;q=0.8',
'Accept-Language': profile.acceptLanguage,
'Accept-Encoding': 'gzip, deflate, br',
}
if (profile.family !== 'safari') {
headers['Sec-Fetch-Dest'] = 'empty'
headers['Sec-Fetch-Mode'] = 'no-cors'
headers['Sec-Fetch-Site'] = 'none'
}
applyChromiumHints(headers, profile)
return headers
}
server/helpers/HostQueue.js
+221
View File
@@ -0,0 +1,221 @@
//
// Per-host politeness scheduler. Knows nothing about scraping.
//
// This is the piece that stops the scraper looking like a crawler. The old code
// did foundUrls.forEach(fetch) - a note with 40 links opened 40 simultaneous
// connections, which is the most bot-like traffic shape available. Here, each
// host gets one request at a time with jittered spacing between them, and a
// global cap bounds total concurrency.
//
const config = require('@config/scrape')
let HostQueue = module.exports = {}
const jitter = (min, max) => Math.round(min + Math.random() * Math.max(0, max - min))
class Queue {
constructor(options = {}) {
this.globalConcurrency = options.globalConcurrency != null
? options.globalConcurrency : config.maxConcurrentFetches
this.perHostConcurrency = options.perHostConcurrency != null
? options.perHostConcurrency : config.perHostConcurrency
this.minSpacingMs = options.minSpacingMs != null
? options.minSpacingMs : config.perHostMinSpacingMs
this.maxSpacingMs = options.maxSpacingMs != null
? options.maxSpacingMs : config.perHostMaxSpacingMs
this.now = options.now || (() => Date.now())
// Injectable so tests with a controlled clock don't spend real
// wall-time waiting out politeness spacing.
this.setTimeoutFn = options.setTimeoutFn || ((fn, ms) => setTimeout(fn, ms))
this.clearTimeoutFn = options.clearTimeoutFn || ((handle) => clearTimeout(handle))
this.pending = [] // FIFO
this.hosts = new Map() // hostKey -> { running, nextAllowedAt }
this.running = 0
this.seq = 0
this.timer = null
this.stopped = false
this.drainWaiters = []
}
hostState(hostKey) {
if (!this.hosts.has(hostKey)) {
this.hosts.set(hostKey, { running: 0, nextAllowedAt: 0 })
}
return this.hosts.get(hostKey)
}
//
// Queue a task for a host. Resolves/rejects with the task's own outcome.
// spacingMs overrides the jitter floor (robots.txt Crawl-delay).
//
schedule(hostKey, taskFn, options = {}) {
if (this.stopped) {
return Promise.reject(new Error('HostQueue stopped'))
}
return new Promise((resolve, reject) => {
this.pending.push({
hostKey: hostKey || '(none)',
taskFn,
resolve,
reject,
spacingMs: options.spacingMs != null ? options.spacingMs : null,
seq: this.seq++,
})
this.pump()
})
}
//
// Run whatever is eligible. FIFO scan rather than per-host round-robin, so
// no host can starve another and order stays predictable.
//
pump() {
if (this.stopped) { return }
this.clearTimer()
while (this.running < this.globalConcurrency) {
const at = this.now()
let index = -1
let earliest = Infinity
for (let i = 0; i < this.pending.length; i++) {
const job = this.pending[i]
const host = this.hostState(job.hostKey)
if (host.running >= this.perHostConcurrency) { continue }
if (host.nextAllowedAt > at) {
earliest = Math.min(earliest, host.nextAllowedAt)
continue
}
index = i
break
}
if (index === -1) {
// Nothing runnable now. Wake up when the earliest host is due.
if (this.pending.length && earliest !== Infinity) {
this.armTimer(earliest - at)
}
break
}
this.run(this.pending.splice(index, 1)[0])
}
this.checkDrained()
}
run(job) {
const host = this.hostState(job.hostKey)
host.running++
this.running++
const settle = () => {
host.running--
this.running--
// Space the NEXT request to this host from the end of this one.
const spacing = job.spacingMs != null
? job.spacingMs
: jitter(this.minSpacingMs, this.maxSpacingMs)
host.nextAllowedAt = this.now() + spacing
this.pump()
}
// A throwing taskFn must not wedge the pump, so the call itself is
// wrapped rather than only its promise.
let result = null
try {
result = Promise.resolve(job.taskFn())
} catch (error) {
result = Promise.reject(error)
}
result.then(
value => { settle(); job.resolve(value) },
error => { settle(); job.reject(error) }
)
}
armTimer(delay) {
this.clearTimer()
this.timer = this.setTimeoutFn(() => {
this.timer = null
this.pump()
}, Math.max(1, delay))
//
// Deliberately NOT unref'd.
//
// This timer is the only thing that will resume a host which is inside
// its politeness gap. unref'ing it means that if nothing else holds the
// event loop open, Node exits and every queued job is silently
// abandoned with its promise never settling. Inside the server that is
// masked by the listening socket, but it makes the queue unusable from
// a standalone script and is a footgun either way.
//
// Shutdown is handled explicitly instead: stop() clears this timer and
// rejects whatever is still queued, and it is wired to SIGINT/SIGTERM
// in server/index.js.
//
}
clearTimer() {
if (this.timer) {
this.clearTimeoutFn(this.timer)
this.timer = null
}
}
size() { return this.pending.length }
inflight() { return this.running }
stats() {
return {
pending: this.pending.length,
running: this.running,
hosts: this.hosts.size,
}
}
checkDrained() {
if (this.pending.length || this.running) { return }
const waiters = this.drainWaiters
this.drainWaiters = []
waiters.forEach(resolve => resolve())
}
// Resolves once the queue is empty and nothing is in flight.
drain() {
if (!this.pending.length && !this.running) { return Promise.resolve() }
return new Promise(resolve => this.drainWaiters.push(resolve))
}
//
// Reject everything still queued. In-flight tasks are not cancelled here -
// their own AbortControllers handle that.
//
stop() {
this.stopped = true
this.clearTimer()
const pending = this.pending
this.pending = []
pending.forEach(job => job.reject(new Error('HostQueue stopped')))
this.checkDrained()
}
}
HostQueue.Queue = Queue
// Shared instance used by the scrape worker. Tests build their own.
HostQueue.shared = new Queue()
server/helpers/HttpFetch.js
+429
View File
@@ -0,0 +1,429 @@
//
// The only module in the server that opens an outbound connection.
//
// Everything that made the old scraper unreliable is handled here, in one place:
// a single AbortController-based timeout (the old code raced request's socket
// timeout against a manual setTimeout that could fire against a null reference
// and crash the process), manual redirect following so the SSRF gate runs on
// every hop, hard size caps enforced while streaming, a Content-Type gate before
// any body is read, real charset decoding, and retries that only fire for
// transient failures.
//
const config = require('@config/scrape')
const UrlSafety = require('@helpers/UrlSafety')
const BrowserProfiles = require('@helpers/BrowserProfiles')
let HttpFetch = module.exports = {}
const HTML_TYPES = ['text/html', 'application/xhtml+xml', 'text/plain', 'application/xml', 'text/xml']
class FetchError extends Error {
constructor(kind, message, extra = {}) {
super(message)
this.name = 'FetchError'
this.kind = kind // timeout|http|blocked|too_large|bad_content_type|redirect_loop|too_many_redirects|network|aborted
this.status = extra.status || null
this.retryAfterMs = extra.retryAfterMs || null
this.url = extra.url || null
this.reason = extra.reason || null
}
}
HttpFetch.FetchError = FetchError
// Injectable so tests don't spend real wall-time in backoff. Far simpler than
// interleaving fake timers with promises.
let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
HttpFetch.__setSleep = (fn) => { sleep = fn }
HttpFetch.__resetSleep = () => { sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) }
const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504, 507, 509, 521, 522, 523, 524])
const isRetryable = (error) => {
if (!(error instanceof FetchError)) { return false }
if (error.kind === 'network' || error.kind === 'timeout') { return true }
if (error.kind === 'http') { return RETRYABLE_STATUS.has(error.status) }
return false
}
// Retry-After is either delta-seconds or an HTTP-date.
const parseRetryAfter = (value) => {
if (!value) { return null }
const seconds = Number(value)
if (Number.isFinite(seconds)) { return Math.max(0, seconds * 1000) }
const when = Date.parse(value)
if (Number.isFinite(when)) { return Math.max(0, when - Date.now()) }
return null
}
const backoffFor = (attempt) => {
const base = Math.min(config.backoffMaxMs, config.backoffBaseMs * Math.pow(2, attempt))
return Math.round(base * (0.5 + Math.random())) // +/-50% jitter
}
const contentTypeOf = (response) => String(response.headers.get('content-type') || '').toLowerCase()
const mimeOf = (contentType) => contentType.split(';')[0].trim()
const charsetFromContentType = (contentType) => {
const match = contentType.match(/charset=\s*"?([^";\s]+)"?/i)
return match ? match[1].toLowerCase() : null
}
// Last resort: sniff a <meta charset> out of the head, read as latin1 so every
// byte maps to a character and the regex can't miss.
const charsetFromMeta = (buffer) => {
const head = buffer.subarray(0, 2048).toString('latin1')
const metaCharset = head.match(/<meta[^>]+charset\s*=\s*["']?\s*([a-z0-9_\-:]+)/i)
if (metaCharset) { return metaCharset[1].toLowerCase() }
const httpEquiv = head.match(/<meta[^>]+content\s*=\s*["'][^"']*charset=\s*([a-z0-9_\-:]+)/i)
if (httpEquiv) { return httpEquiv[1].toLowerCase() }
return null
}
const charsetFromBom = (buffer) => {
if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) { return 'utf-8' }
if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { return 'utf-16le' }
if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) { return 'utf-16be' }
return null
}
const decodeBody = (buffer, declaredCharset) => {
const label = declaredCharset || charsetFromBom(buffer) || charsetFromMeta(buffer) || 'utf-8'
try {
return { text: new TextDecoder(label, { fatal: false }).decode(buffer), charset: label }
} catch (error) {
// Unknown/bogus label - utf-8 with replacement chars beats throwing.
return { text: new TextDecoder('utf-8', { fatal: false }).decode(buffer), charset: 'utf-8' }
}
}
//
// Read the body with a running byte count, aborting mid-stream once the cap is
// passed. Content-Length is checked by the caller too, but it is optional and
// can lie, so this is the cap that actually holds.
//
const readCapped = async (response, maxBytes, controller, url) => {
if (!response.body) { return Buffer.alloc(0) }
const chunks = []
let total = 0
for await (const chunk of response.body) {
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
total += buffer.length
if (total > maxBytes) {
try { controller.abort() } catch (error) { /* already done */ }
throw new FetchError('too_large', `Body exceeded ${maxBytes} bytes`, { url })
}
chunks.push(buffer)
}
return Buffer.concat(chunks, total)
}
//
// One HTTP request, following redirects by hand.
//
// Manual redirects are the whole point: fetch's own 'follow' would hop to
// 127.0.0.1 without ever re-consulting the safety gate.
//
const requestOnce = async (startUrl, options) => {
const {
maxBytes,
acceptMimes,
headersFor,
conditional,
timeoutMs,
deadline,
referer,
} = options
let currentUrl = typeof startUrl === 'string' ? startUrl : startUrl.href
const visited = new Set()
for (let hop = 0; hop <= config.maxRedirects; hop++) {
if (visited.has(currentUrl)) {
throw new FetchError('redirect_loop', `Redirect loop at ${currentUrl}`, { url: currentUrl })
}
visited.add(currentUrl)
// Re-validate on every hop. This is the check the old code never had.
try {
await UrlSafety.assertFetchable(currentUrl)
} catch (error) {
// A DNS failure is transient - the host may simply be down right
// now - so it must not be classified alongside "this address is
// private", which is a permanent refusal. Collapsing the two meant
// a briefly-unresolvable domain got parked for the full cache TTL
// and had its attachment row deleted.
const kind = error.reason === 'dns_failed' ? 'network' : 'blocked'
throw new FetchError(kind, error.message, { url: currentUrl, reason: error.reason })
}
const remaining = deadline - Date.now()
if (remaining <= 0) {
throw new FetchError('timeout', 'Total budget exhausted', { url: currentUrl })
}
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, remaining))
let response = null
try {
const headers = headersFor(currentUrl, referer)
if (conditional && conditional.etag) { headers['If-None-Match'] = conditional.etag }
if (conditional && conditional.lastModified) { headers['If-Modified-Since'] = conditional.lastModified }
response = await fetch(currentUrl, {
method: 'GET',
redirect: 'manual',
signal: controller.signal,
headers,
})
} catch (error) {
clearTimeout(timer)
if (controller.signal.aborted) {
throw new FetchError('timeout', `Timed out after ${timeoutMs}ms`, { url: currentUrl })
}
throw new FetchError('network', error.message || 'Network failure', { url: currentUrl })
}
// Redirect?
if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
clearTimeout(timer)
try { if (response.body) { await response.body.cancel() } } catch (error) { /* nothing to drain */ }
const location = response.headers.get('location')
let next = null
try {
next = new URL(location, currentUrl)
} catch (error) {
throw new FetchError('network', `Unparseable redirect target ${location}`, { url: currentUrl })
}
// Never follow https -> http. A downgrade mid-chain is either an
// attack or a misconfiguration, and is not worth the preview.
if (new URL(currentUrl).protocol === 'https:' && next.protocol === 'http:') {
throw new FetchError('blocked', `Refusing https->http downgrade to ${next.href}`, { url: currentUrl })
}
currentUrl = next.href
continue
}
clearTimeout(timer)
if (response.status === 304) {
try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
return { notModified: true, status: 304, finalUrl: currentUrl, response }
}
if (!response.ok) {
try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
throw new FetchError('http', `HTTP ${response.status} for ${currentUrl}`, {
status: response.status,
retryAfterMs: parseRetryAfter(response.headers.get('retry-after')),
url: currentUrl,
})
}
const contentType = contentTypeOf(response)
const mime = mimeOf(contentType)
// Gate on type BEFORE reading a byte. This is what stops a 100MB binary
// being handed to cheerio.
if (acceptMimes && !acceptMimes(mime)) {
try { controller.abort() } catch (error) { /* ignore */ }
throw new FetchError('bad_content_type', `Refusing content-type ${mime || '(none)'}`, { url: currentUrl })
}
const declaredLength = Number(response.headers.get('content-length'))
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
try { controller.abort() } catch (error) { /* ignore */ }
throw new FetchError('too_large', `Declared length ${declaredLength} over cap`, { url: currentUrl })
}
const buffer = await readCapped(response, maxBytes, controller, currentUrl)
return {
notModified: false,
status: response.status,
finalUrl: currentUrl,
contentType,
mime,
buffer,
response,
}
}
throw new FetchError('too_many_redirects', `More than ${config.maxRedirects} redirects`, { url: currentUrl })
}
//
// Retry wrapper. Only transient classes are retried - a 404 is an answer.
//
const withRetries = async (attemptFn, { retries, deadline }) => {
let lastError = null
for (let attempt = 0; attempt <= retries; attempt++) {
try {
return await attemptFn()
} catch (error) {
lastError = error
if (!isRetryable(error) || attempt === retries) { throw error }
// A server asking us to come back in an hour is not something to
// sit on - surface it so the cache can park the URL instead.
if (error.retryAfterMs && error.retryAfterMs > config.maxRetryAfterMs) { throw error }
const wait = error.retryAfterMs != null
? Math.min(error.retryAfterMs, config.maxRetryAfterMs)
: backoffFor(attempt)
if (Date.now() + wait >= deadline) { throw error }
await sleep(wait)
}
}
throw lastError
}
//
// Fetch an HTML document.
//
HttpFetch.fetchDocument = async (url, options = {}) => {
const timeoutMs = options.timeoutMs || config.pageTimeoutMs
const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
const maxBytes = options.maxBytes || config.maxHtmlBytes
const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
const result = await withRetries(() => requestOnce(url, {
maxBytes,
acceptMimes: (mime) => HTML_TYPES.includes(mime),
headersFor: (target, referer) => BrowserProfiles.documentHeaders(profile, {
url: target,
// A browser arriving at a page has *some* referer; its own origin is
// the least surprising one and pairs with Sec-Fetch-Site: same-origin.
referer: referer || originOf(target),
}),
conditional: options.conditional,
timeoutMs,
deadline,
referer: options.referer,
}), { retries: options.retries != null ? options.retries : config.retries, deadline })
if (result.notModified) {
return {
notModified: true,
status: 304,
finalUrl: result.finalUrl,
html: null,
profileId: profile.id,
}
}
const declaredCharset = charsetFromContentType(result.contentType)
const { text, charset } = decodeBody(result.buffer, declaredCharset)
return {
notModified: false,
status: result.status,
finalUrl: result.finalUrl,
html: text,
charset,
contentType: result.contentType,
bytes: result.buffer.length,
etag: result.response.headers.get('etag') || null,
lastModified: result.response.headers.get('last-modified') || null,
profileId: profile.id,
}
}
//
// Fetch a binary (thumbnail image).
//
HttpFetch.fetchBinary = async (url, options = {}) => {
const timeoutMs = options.timeoutMs || config.imageTimeoutMs
const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
const maxBytes = options.maxBytes || config.maxImageBytes
const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
const acceptMimes = (mime) => {
if (!mime.startsWith('image/')) { return false }
if (mime === 'image/svg+xml' && !config.allowSvgThumbnails) { return false }
return true
}
const result = await withRetries(() => requestOnce(url, {
maxBytes,
acceptMimes,
headersFor: (target) => BrowserProfiles.imageHeaders(profile, {
url: target,
pageUrl: options.pageUrl,
}),
timeoutMs,
deadline,
}), { retries: options.retries != null ? options.retries : config.retries, deadline })
return {
status: result.status,
finalUrl: result.finalUrl,
contentType: result.contentType,
mime: result.mime,
buffer: result.buffer,
profileId: profile.id,
}
}
//
// Fetch robots.txt. Short leash, no retries - fail-open is handled by the caller.
//
HttpFetch.fetchText = async (url, options = {}) => {
const timeoutMs = options.timeoutMs || 5000
const deadline = Date.now() + (options.budgetMs || timeoutMs + 1000)
const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
const result = await requestOnce(url, {
maxBytes: options.maxBytes || config.maxRobotsBytes,
acceptMimes: null, // robots.txt is served as every type under the sun
headersFor: () => BrowserProfiles.robotsHeaders(profile),
timeoutMs,
deadline,
})
if (result.notModified) { return { status: 304, text: null, finalUrl: result.finalUrl } }
const { text } = decodeBody(result.buffer, charsetFromContentType(result.contentType))
return { status: result.status, text, finalUrl: result.finalUrl }
}
const hostnameOf = (url) => {
try { return new URL(typeof url === 'string' ? url : url.href).hostname }
catch (error) { return '' }
}
const originOf = (url) => {
try { return new URL(url).origin + '/' }
catch (error) { return undefined }
}
//
// Deliberately NOT attempted here, so nobody wonders later:
// - Header ordering: fetch normalises it; controlling order needs raw sockets.
// - HTTP/2: undici's fetch is h1 unless an Agent with allowH2 is supplied.
// - TLS/JA3 fingerprinting and proxy rotation: ruled out, we are being polite
// rather than evasive.
//
server/helpers/Realtime.js
+43
View File
@@ -0,0 +1,43 @@
//
// Socket.IO emit wrapper.
//
// Replaces the `if(typeof SocketIo != 'undefined')` guards that were scattered
// through the models. Those guards did not just avoid a crash - they silently
// disabled ALL scraping under Jest, which is why none of this code was ever
// testable. Here the no-socket case is a no-op rather than a skipped branch, so
// the surrounding logic runs identically in tests and in production.
//
let Realtime = module.exports = {}
let warned = false
const server = () => (typeof global.SocketIo !== 'undefined' && global.SocketIo) ? global.SocketIo : null
Realtime.isEnabled = () => server() !== null
const emit = (room, event, payload) => {
const io = server()
if (!io) {
if (!warned) {
warned = true
console.log('[realtime] no socket server bound; emits are no-ops')
}
return false
}
try {
if (payload === undefined) { io.to(room).emit(event) }
else { io.to(room).emit(event, payload) }
return true
} catch (error) {
console.log('[realtime] emit failed', event, error.message)
return false
}
}
Realtime.emitToUser = (userId, event, payload) => emit(`user:${userId}`, event, payload)
Realtime.emitToNote = (noteRawTextId, event, payload) => emit(`note:${noteRawTextId}`, event, payload)
Realtime.__resetWarning = () => { warned = false }
server/helpers/Robots.js
+216
View File
@@ -0,0 +1,216 @@
//
// robots.txt fetching, parsing and caching.
//
// Posture: polite, not evasive. An explicit Disallow is respected (fail-closed);
// a robots.txt we cannot retrieve is treated as permissive (fail-open), because
// a 404 or a flaky server is not a refusal and these are single-page fetches of
// URLs a user explicitly saved.
//
const config = require('@config/scrape')
const HttpFetch = require('@helpers/HttpFetch')
let Robots = module.exports = {}
// origin -> { rules, expiresAt }
const cache = new Map()
Robots.__resetCache = () => cache.clear()
Robots.__cacheSize = () => cache.size
const ALLOW_ALL = { groups: [], allowAll: true }
//
// Parse robots.txt into user-agent groups.
//
// Consecutive User-agent lines share one rule block, which is the part naive
// parsers get wrong.
//
Robots.parse = (text) => {
if (!text || typeof text !== 'string') { return ALLOW_ALL }
const groups = []
let current = null
let expectingAgents = false
for (const rawLine of text.split(/\r?\n/)) {
const line = rawLine.replace(/#.*$/, '').trim()
if (!line) { continue }
const separator = line.indexOf(':')
if (separator === -1) { continue }
const field = line.slice(0, separator).trim().toLowerCase()
const value = line.slice(separator + 1).trim()
if (field === 'user-agent') {
// A User-agent line directly after another starts a shared group.
if (!current || !expectingAgents) {
current = { agents: [], rules: [], crawlDelayMs: null }
groups.push(current)
}
current.agents.push(value.toLowerCase())
expectingAgents = true
continue
}
if (!current) { continue } // directive before any user-agent
expectingAgents = false
if (field === 'disallow' || field === 'allow') {
// "Disallow:" with an empty value means allow everything.
if (field === 'disallow' && value === '') { continue }
current.rules.push({ allow: field === 'allow', path: value })
continue
}
if (field === 'crawl-delay') {
const seconds = Number(value)
if (Number.isFinite(seconds) && seconds >= 0) {
current.crawlDelayMs = Math.round(seconds * 1000)
}
}
}
return { groups, allowAll: groups.length === 0 }
}
//
// Pick the group whose agent token best matches us. A named match always beats
// the "*" wildcard, regardless of file order.
//
const groupFor = (rules, userAgentToken) => {
if (!rules || rules.allowAll) { return null }
const token = userAgentToken.toLowerCase()
let best = null
let bestScore = -1
for (const group of rules.groups) {
for (const agent of group.agents) {
let score = -1
if (agent === '*') { score = 0 }
else if (token.includes(agent)) { score = agent.length }
if (score > bestScore) {
bestScore = score
best = group
}
}
}
return best
}
//
// Match a robots path pattern, supporting the * and $ wildcards.
//
const matchesPattern = (pattern, path) => {
if (pattern === '') { return false }
if (!pattern.includes('*') && !pattern.includes('$')) {
return path.startsWith(pattern)
}
const anchored = pattern.endsWith('$')
const body = anchored ? pattern.slice(0, -1) : pattern
const expression = body
.split('*')
.map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
.join('.*')
return new RegExp('^' + expression + (anchored ? '$' : '')).test(path)
}
//
// Longest matching rule wins; Allow beats Disallow at equal length.
//
Robots.isPathAllowed = (rules, path, userAgentToken) => {
const group = groupFor(rules, userAgentToken)
if (!group) { return true }
let decision = true
let decisionLength = -1
for (const rule of group.rules) {
if (!matchesPattern(rule.path, path)) { continue }
const length = rule.path.length
if (length > decisionLength || (length === decisionLength && rule.allow)) {
decision = rule.allow
decisionLength = length
}
}
return decision
}
const evict = () => {
if (cache.size <= config.robotsMaxOrigins) { return }
// Map preserves insertion order, so the oldest key is first.
const oldest = cache.keys().next().value
cache.delete(oldest)
}
const getRules = async (origin) => {
const cached = cache.get(origin)
if (cached && cached.expiresAt > Date.now()) { return cached.rules }
let rules = ALLOW_ALL
let ttl = config.robotsTtlMs
try {
const result = await HttpFetch.fetchText(`${origin}/robots.txt`, {
maxBytes: config.maxRobotsBytes,
})
rules = Robots.parse(result.text)
} catch (error) {
// No robots.txt, or we could not reach it. Permissive, but cached for a
// short while only so a transient outage does not grant a day's pass.
rules = ALLOW_ALL
ttl = (error && error.kind === 'http' && error.status === 404)
? config.robotsNegativeTtlMs
: config.robotsErrorTtlMs
}
cache.set(origin, { rules, expiresAt: Date.now() + ttl })
evict()
return rules
}
Robots.__getRules = getRules
const originAndPath = (url) => {
const parsed = new URL(typeof url === 'string' ? url : url.href)
return { origin: parsed.origin, path: parsed.pathname + parsed.search }
}
//
// Is this URL allowed by its host's robots.txt?
//
Robots.isAllowed = async (url, userAgentToken) => {
let parts = null
try { parts = originAndPath(url) } catch (error) { return false }
const rules = await getRules(parts.origin)
return Robots.isPathAllowed(rules, parts.path, userAgentToken || config.robotsUaToken)
}
//
// Crawl-delay for this host, or null. Capped: a site asking for an hour between
// requests gets its job failed rather than parked in the queue forever.
//
Robots.crawlDelayMs = async (url, userAgentToken) => {
let parts = null
try { parts = originAndPath(url) } catch (error) { return null }
const rules = await getRules(parts.origin)
const group = groupFor(rules, (userAgentToken || config.robotsUaToken).toLowerCase())
if (!group || group.crawlDelayMs == null) { return null }
return group.crawlDelayMs
}
Robots.exceedsCrawlDelayCap = (delayMs) => {
return delayMs != null && delayMs > config.maxCrawlDelayMs
}
server/helpers/Schema.js
+239
View File
@@ -0,0 +1,239 @@
//
// Verification of the scraper's schema and privileges. Issues NO DDL.
//
// This module used to converge the schema at boot with CREATE TABLE / ALTER
// TABLE. That worked only because dev connects as root (shell.nix exports
// DB_USER=root); on production, which runs as a least-privilege `app` user, it
// failed with:
//
// CREATE command denied to user 'app'@'localhost' for table scrape_cache
//
// The restricted account is correct - the server ingests untrusted URLs and
// should not be able to reshape its own tables. So DDL moved out of the
// application entirely: an admin runs ./migrateDatabase.sh once, and this
// module only checks that the result is present and usable.
//
// Note on why the DDL is REMOVED rather than guarded by an existence check:
// MariaDB evaluates the CREATE privilege before IF NOT EXISTS, so
// `CREATE TABLE IF NOT EXISTS` would keep erroring for a DML-only user on every
// boot even once the table exists.
//
const db = require('@config/database')
let Schema = module.exports = {}
const REQUIRED_TABLE = 'scrape_cache'
const REQUIRED_COLUMNS = ['scrape_state', 'scrape_attempts', 'enqueued_at']
const REQUIRED_INDEX = 'attachment_scrape_state'
//
// The FULLTEXT index backing Attachment.textSearch's MATCH..AGAINST.
//
// Checked by TYPE, not just existence: Prisma's `@@fulltext` is a preview
// feature and MariaDB is not among its documented databases, so a future
// migration could plausibly emit a plain BTREE index of the same name. That
// would make attachment search return nothing at all, with no error anywhere -
// so it is worth asserting the type explicitly on every boot.
//
const REQUIRED_FULLTEXT = { table: 'attachment', index: 'attachment_text_index' }
// The scraper writes and deletes, not just reads. DELETE is genuinely needed:
// terminal failures drop the shell attachment row (ScrapeQueue), and
// ScrapeCache.purgeExpired removes expired rows.
const REQUIRED_PRIVILEGES = ['SELECT', 'INSERT', 'UPDATE', 'DELETE']
// Set by verifyScrapeSchema so the per-request paths can check a flag rather
// than querying information_schema on every note save.
let ready = false
let lastResult = null
Schema.isScrapeSchemaReady = () => ready
Schema.lastVerification = () => lastResult
// Test seam.
Schema.__setReady = (value) => { ready = !!value }
Schema.__reset = () => { ready = false; lastResult = null }
const tableExists = async (table) => {
const [rows] = await db.promise().query(
`SELECT 1 FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? LIMIT 1`,
[table]
)
return rows.length > 0
}
const columnExists = async (table, column) => {
const [rows] = await db.promise().query(
`SELECT 1 FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`,
[table, column]
)
return rows.length > 0
}
//
// Does this index exist AND is it actually FULLTEXT?
//
const fulltextIndexExists = async (table, index) => {
const [rows] = await db.promise().query(
`SELECT 1 FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?
AND INDEX_TYPE = 'FULLTEXT' LIMIT 1`,
[table, index]
)
return rows.length > 0
}
const indexExists = async (table, index) => {
const [rows] = await db.promise().query(
`SELECT 1 FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1`,
[table, index]
)
return rows.length > 0
}
Schema.__tableExists = tableExists
Schema.__columnExists = columnExists
Schema.__indexExists = indexExists
Schema.__fulltextIndexExists = fulltextIndexExists
//
// Parse SHOW GRANTS output into the set of privileges that apply to this
// database. Any user may run SHOW GRANTS FOR CURRENT_USER(), so this needs no
// privileges of its own.
//
// Grant lines look like:
// GRANT ALL PRIVILEGES ON *.* TO `root`@`localhost` WITH GRANT OPTION
// GRANT SELECT, INSERT, UPDATE, DELETE ON `application`.* TO `app`@`localhost`
// GRANT SELECT ON `application`.`attachment` TO `app`@`localhost`
// GRANT USAGE ON *.* TO `app`@`localhost`
//
const PRIVILEGE_PATTERN = /^GRANT\s+(.+?)\s+ON\s+(\S+)\s+TO\s/i
Schema.parseGrants = (lines, database, tables) => {
const granted = new Set()
// Scope strings that cover our tables. Backticks are optional in output.
const strip = (value) => value.replace(/`/g, '')
const relevantScopes = new Set([
'*.*',
`${database}.*`,
...tables.map(table => `${database}.${table}`),
])
for (const line of lines) {
const match = String(line).match(PRIVILEGE_PATTERN)
if (!match) { continue }
const scope = strip(match[2])
if (!relevantScopes.has(scope)) { continue }
// Drop any column-level qualifiers, e.g. "SELECT (id, url)".
const privileges = match[1].replace(/\s*\([^)]*\)/g, '')
for (const privilege of privileges.split(',')) {
const name = privilege.trim().toUpperCase()
if (!name || name === 'USAGE') { continue }
if (name === 'ALL PRIVILEGES' || name === 'ALL') {
REQUIRED_PRIVILEGES.forEach(required => granted.add(required))
continue
}
granted.add(name)
}
}
return granted
}
const checkPrivileges = async () => {
let lines = []
try {
const [rows] = await db.promise().query('SHOW GRANTS FOR CURRENT_USER()')
// Each row is an object with a single, server-named column.
lines = rows.map(row => Object.values(row)[0])
} catch (error) {
// If we cannot read our own grants, do not block on a guess - the
// object checks above are the real gate.
console.log('[schema] could not read grants, skipping privilege check:', error.message)
return { missing: [], database: null, currentUser: null }
}
const [dbRows] = await db.promise().query('SELECT DATABASE() AS name, CURRENT_USER() AS who')
const database = dbRows[0] && dbRows[0].name
const currentUser = dbRows[0] && dbRows[0].who
const granted = Schema.parseGrants(lines, database, ['attachment', REQUIRED_TABLE])
return {
missing: REQUIRED_PRIVILEGES.filter(privilege => !granted.has(privilege)),
database,
currentUser,
}
}
// 'app@localhost' -> "'app'@'localhost'", so the remedy is copy-pasteable.
const quoteUser = (currentUser) => {
if (!currentUser) { return '<app user>' }
const [name, host] = String(currentUser).split('@')
return `'${name}'@'${host || 'localhost'}'`
}
//
// Check that everything the scraper needs is present. Never mutates anything.
//
// Returns { ok, missingObjects, missingPrivileges, remedy }.
//
Schema.verifyScrapeSchema = async () => {
const missingObjects = []
if (!await tableExists(REQUIRED_TABLE)) {
missingObjects.push(`table ${REQUIRED_TABLE}`)
}
for (const column of REQUIRED_COLUMNS) {
if (!await columnExists('attachment', column)) {
missingObjects.push(`attachment.${column}`)
}
}
if (!await indexExists('attachment', REQUIRED_INDEX)) {
missingObjects.push(`attachment index ${REQUIRED_INDEX}`)
}
if (!await fulltextIndexExists(REQUIRED_FULLTEXT.table, REQUIRED_FULLTEXT.index)) {
missingObjects.push(`FULLTEXT index ${REQUIRED_FULLTEXT.table}.${REQUIRED_FULLTEXT.index}`)
}
const privileges = await checkPrivileges()
const missingPrivileges = privileges.missing || []
const remedy = []
if (missingObjects.length) {
remedy.push('run ./migrateDatabase.sh as a database admin')
}
if (missingPrivileges.length) {
// Name the real database and user so this can be pasted straight into
// a mysql prompt rather than decoded first.
const target = `${privileges.database || '<database>'}.*`
remedy.push(
`GRANT ${missingPrivileges.join(', ')} ON ${target} TO ${quoteUser(privileges.currentUser)}`
)
}
lastResult = {
ok: missingObjects.length === 0 && missingPrivileges.length === 0,
missingObjects,
missingPrivileges,
remedy: remedy.join('; '),
}
ready = lastResult.ok
return lastResult
}
server/helpers/SiteScrape.js
+362 -135
View File
@@ -1,8 +1,15 @@
let SiteScrape = module.exports = {}
//
// Pure parsing of an already-fetched page. No network I/O, no logging.
//
// $ = the cheerio scrape object
//
// Everything here is a pure function of ($, url), which is what makes it
// testable against saved HTML fixtures instead of the live internet.
//
const config = require('@config/scrape')
let SiteScrape = module.exports = {}
const removeWhitespace = /\s+/g
@@ -13,162 +20,311 @@ const commonWords = ['just','start','what','these','how', 'was', 'being','can','
'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
SiteScrape.getTitle = ($) => {
let title = $('title').text().replace(removeWhitespace, " ")
return title
const clean = (value, maxLength) => {
if (value == null) { return '' }
const text = String(value).replace(removeWhitespace, ' ').trim()
return maxLength ? text.substring(0, maxLength) : text
}
//Finds all urls in text, removes duplicates, makes sure they have https://
SiteScrape.getCleanUrls = (textBlock) => {
//Find all URLs in text
//@TODO - Use the process text library for this function
const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
let allUrls = textBlock.match(urlPattern)
// First non-empty wins. Each candidate is a thunk so a missing branch costs
// nothing and nothing throws on the way to the answer.
const firstOf = (candidates) => {
for (const candidate of candidates) {
let value = ''
try { value = clean(typeof candidate === 'function' ? candidate() : candidate) }
catch (error) { value = '' }
if (value) { return value }
}
return ''
}
if(allUrls == null){
return []
}
const metaContent = ($, selector) => {
const node = $(selector).first()
return node.length ? node.attr('content') : ''
}
//Every URL needs HTTPS!!!
let foundUrls = []
allUrls.forEach( (item, index) => {
//add protocol if it is missing
if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
allUrls[index] = 'https://'+item
}
//convert http to https
if(item.indexOf('http://') >= 0){
allUrls[index] = item.replace('http://','https://')
//
// JSON-LD. Malformed JSON-LD is extremely common, so every parse is guarded
// and a bad block is skipped rather than failing the whole page.
//
const LD_TYPES = ['Article', 'NewsArticle', 'BlogPosting', 'WebPage', 'Product', 'VideoObject', 'Recipe']
SiteScrape.getJsonLd = ($) => {
const blocks = []
$('script[type="application/ld+json"]').each((index, element) => {
const raw = $(element).contents().text()
if (!raw || !raw.trim()) { return }
let parsed = null
try { parsed = JSON.parse(raw) } catch (error) { return }
// A block may be an object, an array, or wrap its payload in @graph.
const flatten = (node) => {
if (!node || typeof node !== 'object') { return }
if (Array.isArray(node)) { return node.forEach(flatten) }
if (Array.isArray(node['@graph'])) { node['@graph'].forEach(flatten) }
blocks.push(node)
}
flatten(parsed)
})
//Remove all duplicates
foundUrls = [...new Set(allUrls)]
if (!blocks.length) { return null }
return foundUrls
// Prefer a block whose @type looks like page content.
const typed = blocks.find(block => {
const type = block['@type']
const types = Array.isArray(type) ? type : [type]
return types.some(entry => LD_TYPES.includes(entry))
})
return typed || blocks[0]
}
const jsonLdImage = (block) => {
if (!block || !block.image) { return '' }
const image = Array.isArray(block.image) ? block.image[0] : block.image
if (!image) { return '' }
if (typeof image === 'string') { return image }
if (typeof image === 'object') { return image.url || image.contentUrl || '' }
return ''
}
//
// Page title.
//
SiteScrape.getTitle = ($) => {
const ld = SiteScrape.getJsonLd($)
return firstOf([
() => metaContent($, 'meta[property="og:title"]'),
() => metaContent($, 'meta[name="twitter:title"]'),
() => ld && (ld.headline || ld.name),
() => $('title').first().text(),
() => $('h1').first().text(),
]).substring(0, 500)
}
//
// Page description. The old parser extracted none at all.
//
SiteScrape.getDescription = ($) => {
const ld = SiteScrape.getJsonLd($)
return firstOf([
() => metaContent($, 'meta[property="og:description"]'),
() => metaContent($, 'meta[name="twitter:description"]'),
() => metaContent($, 'meta[name="description"]'),
() => ld && ld.description,
]).substring(0, 1000)
}
SiteScrape.getSiteName = ($, url) => {
return firstOf([
() => metaContent($, 'meta[property="og:site_name"]'),
() => metaContent($, 'meta[name="application-name"]'),
() => {
try { return new URL(url).hostname.replace(/^www\./, '') }
catch (error) { return '' }
},
]).substring(0, 255)
}
//
// Canonical URL. Used as the base for resolving relative asset paths, which is
// what the old manual hostname-concatenation got wrong.
//
SiteScrape.getCanonicalUrl = ($, base) => {
const candidate = firstOf([
() => $('link[rel="canonical"]').first().attr('href'),
() => metaContent($, 'meta[property="og:url"]'),
])
if (!candidate) { return base || '' }
try { return new URL(candidate, base || undefined).href }
catch (error) { return base || '' }
}
//Site hostname with https:// eg: https://www.google.com
SiteScrape.getHostName = (url) => {
try {
return 'https://' + (new URL(url)).hostname
} catch (error) {
return ''
}
}
var hostname = 'https://'+(new URL(url)).hostname;
// console.log('hostname', hostname)
return hostname
//
// Image selection.
//
// Extension allowlist plus an allow-if-extensionless rule. The old substring
// filter required .jpg/.jpeg/.png/.gif anywhere in the string, which silently
// dropped .webp, .avif, .ico and - more importantly - every extensionless
// og:image endpoint like /og?title=x, which is what most modern sites serve.
//
const IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.avif', '.ico', '.bmp', '.jfif']
const TRACKING_PIXEL = /(^|[\/_\-.])(1x1|pixel|spacer|blank|transparent|dot)\.(gif|png)/i
const acceptableImagePath = (href) => {
let pathname = ''
try { pathname = new URL(href).pathname.toLowerCase() }
catch (error) { return false }
const lastSegment = pathname.split('/').pop() || ''
const dot = lastSegment.lastIndexOf('.')
// No extension at all - allow it. An og:image endpoint is the common case.
if (dot === -1) { return true }
const extension = lastSegment.slice(dot)
if (extension === '.svg') { return !!config.allowSvgThumbnails }
return IMAGE_EXTENSIONS.includes(extension)
}
// Resolve against the page's canonical base. Handles protocol-relative
// (//host/x), root-relative, and path-relative uniformly - no string surgery.
const resolveImage = (src, base) => {
if (!src) { return '' }
const value = String(src).trim()
if (!value || value.startsWith('data:')) { return '' }
try { return new URL(value, base).href }
catch (error) { return '' }
}
// Largest candidate in a srcset, by width descriptor.
const largestFromSrcset = (srcset) => {
if (!srcset) { return '' }
let best = ''
let bestWidth = -1
for (const entry of String(srcset).split(',')) {
const parts = entry.trim().split(/\s+/)
if (!parts[0]) { continue }
const descriptor = parts[1] || ''
const width = descriptor.endsWith('w') ? parseInt(descriptor, 10) : 0
if (width > bestWidth) {
bestWidth = width
best = parts[0]
}
}
return best
}
const iconCandidates = ($) => {
const icons = []
// apple-touch-icon is usually the largest and cleanest of the icons.
$('link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"]').each((index, element) => {
const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
icons.push({ href: $(element).attr('href'), weight: 1000 + sizes })
})
$('link[rel="icon"], link[rel="shortcut icon"]').each((index, element) => {
const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
icons.push({ href: $(element).attr('href'), weight: sizes })
})
return icons
.sort((a, b) => b.weight - a.weight)
.map(icon => icon.href)
}
const inlineImageCandidates = ($) => {
const images = []
$('img').each((index, element) => {
const node = $(element)
// Guarded: the old code read element.attribs.src directly and threw on
// an <img> with no attributes.
const src = node.attr('src') || node.attr('data-src') || largestFromSrcset(node.attr('srcset'))
if (!src) { return }
const width = parseInt(node.attr('width') || '0', 10) || 0
const height = parseInt(node.attr('height') || '0', 10) || 0
// Tracking pixels and spacers.
if ((width > 0 && width <= 1) || (height > 0 && height <= 1)) { return }
if (TRACKING_PIXEL.test(src)) { return }
images.push({ src, area: width * height, order: index })
})
// Biggest first; ties keep document order, so the top of the page wins.
return images
.sort((a, b) => (b.area - a.area) || (a.order - b.order))
.map(image => image.src)
}
//
// Ordered candidate list, so "which image" is an explicit priority rather than
// the old reverse-iterate-and-overwrite loop that only worked by accident.
//
SiteScrape.getImageCandidates = ($, url) => {
const base = SiteScrape.getCanonicalUrl($, url) || url
const ld = SiteScrape.getJsonLd($)
const raw = [
metaContent($, 'meta[property="og:image:secure_url"]'),
metaContent($, 'meta[property="og:image:url"]'),
metaContent($, 'meta[property="og:image"]'),
metaContent($, 'meta[name="twitter:image"]'),
metaContent($, 'meta[name="twitter:image:src"]'),
jsonLdImage(ld),
...inlineImageCandidates($),
...iconCandidates($),
]
const seen = new Set()
const candidates = []
for (const entry of raw) {
const resolved = resolveImage(entry, base)
if (!resolved || seen.has(resolved)) { continue }
if (!acceptableImagePath(resolved)) { continue }
seen.add(resolved)
candidates.push(resolved)
}
return candidates
}
// URL for image that can be downloaded to represent website
SiteScrape.getDisplayImage = ($, url) => {
const hostname = SiteScrape.getHostName(url)
let metaImg = $('[property="og:image"]')
let shortcutIcon = $('[rel="shortcut icon"]')
let favicon = $('[rel="icon"]')
let randomImg = $('img')
//Set of images we may want gathered from various places in source
let imagesWeWant = []
let thumbnail = ''
//Scrape metadata for page image
if(randomImg && randomImg.length > 0){
let imgSrcs = []
for (let i = 0; i < randomImg.length; i++) {
imgSrcs.push( randomImg[i].attribs.src )
}
const half = Math.ceil(imgSrcs.length / 2)
imagesWeWant = [...imgSrcs.slice(-half), ...imgSrcs.slice(0,half) ]
}
//Grab the shortcut icon
if(favicon && favicon[0] && favicon[0].attribs){
imagesWeWant.push(favicon[0].attribs.href)
}
//Grab the shortcut icon
if(shortcutIcon && shortcutIcon[0] && shortcutIcon[0].attribs){
imagesWeWant.push(shortcutIcon[0].attribs.href)
}
//Grab the presentation image for the site
if(metaImg && metaImg[0] && metaImg[0].attribs){
imagesWeWant.unshift(metaImg[0].attribs.content)
}
// console.log(imagesWeWant)
//Remove everything that isn't an accepted file format
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
let img = String(imagesWeWant[i])
if(
!img.includes('.jpg') &&
!img.includes('.jpeg') &&
!img.includes('.png') &&
!img.includes('.gif')
){
imagesWeWant.splice(i,1)
}
}
//Find if we have absolute thumbnails or not
let foundAbsolute = false
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
let img = imagesWeWant[i]
//Add host name if its not included
if(String(img).includes('//') || String(img).includes('http')){
foundAbsolute = true
break
}
}
//Go through all found images. Grab the one closest to the top. Closer is better
for (let i = imagesWeWant.length - 1; i >= 0; i--) {
let img = imagesWeWant[i]
if(!String(img).includes('//') && foundAbsolute){
continue;
}
//Only add host to images if no absolute images were found
if(!String(img).includes('//') ){
if(img.indexOf('/') != 0){
img = '/' + img
}
img = hostname + img
}
if(img.indexOf('//') == 0){
img = 'https:' + img //Scrape breaks without protocol
}
thumbnail = img
}
return thumbnail
const candidates = SiteScrape.getImageCandidates($, url)
return candidates.length ? candidates[0] : ''
}
// Get all the site text and parse out the words that appear most
SiteScrape.getKeywords = ($) => {
let majorContent = ''
// Strip chrome and scripts before counting, otherwise navigation labels and
// inline JS identifiers dominate the frequency table.
const working = $.root().clone()
working.find('script, style, noscript, nav, header, footer, aside, form, svg').remove()
majorContent += $('[class*=content]').text()
.replace(removeWhitespace, " ") //Remove all whitespace
// .replace(/\W\s/g, '') //Remove all non alphanumeric characters
.substring(0,6000) //Limit to 6000 characters
let source = working.find('article, main, [role=main], [class*=content]').text()
if (!source || source.trim().length < 200) {
source = working.find('body').text() || working.text()
}
const majorContent = String(source)
.replace(removeWhitespace, ' ')
.substring(0, config.maxKeywordSourceChars)
.toLowerCase()
.replace(/[^A-Za-z0-9- ]/g, '');
console.log(majorContent)
.replace(/[^A-Za-z0-9- ]/g, '')
//Count frequency of each word in scraped text
let frequency = {}
@@ -213,7 +369,78 @@ SiteScrape.getKeywords = ($) => {
return ''
}
SiteScrape.getMainText = ($) => {}
//
// The plaintext blob stored in attachment.text and indexed by the MySQL
// FULLTEXT index. Must stay unencrypted for MATCH ... AGAINST to work.
//
SiteScrape.buildSearchText = ({ title, description, siteName, keywords }) => {
const parts = []
if (title) { parts.push(title) }
if (description) { parts.push(description) }
if (siteName) { parts.push(siteName) }
if (keywords) { parts.push(keywords) }
return parts.join('\n').substring(0, config.maxSearchTextBytes)
}
//
// One call, everything a link preview needs.
//
SiteScrape.parse = ($, url) => {
const canonicalUrl = SiteScrape.getCanonicalUrl($, url)
const title = SiteScrape.getTitle($)
const description = SiteScrape.getDescription($)
const siteName = SiteScrape.getSiteName($, url)
const keywords = SiteScrape.getKeywords($)
const imageCandidates = SiteScrape.getImageCandidates($, url)
return {
title,
description,
siteName,
canonicalUrl,
keywords,
imageUrl: imageCandidates.length ? imageCandidates[0] : '',
imageCandidates,
hostname: SiteScrape.getHostName(url),
searchText: SiteScrape.buildSearchText({ title, description, siteName, keywords }),
}
}
//
// Finds all urls in text, removes duplicates, makes sure they have https://
//
// DELIBERATELY UNCHANGED. This decides which URLs are detected in note text and
// what string lands in attachment.url, so altering it would change link
// behaviour for every existing note - explicitly out of scope for the scraper
// rewrite. The @TODO below is left in place for the same reason.
//
SiteScrape.getCleanUrls = (textBlock) => {
//Find all URLs in text
//@TODO - Use the process text library for this function
const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
let allUrls = textBlock.match(urlPattern)
if(allUrls == null){
return []
}
//Every URL needs HTTPS!!!
let foundUrls = []
allUrls.forEach( (item, index) => {
//add protocol if it is missing
if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
allUrls[index] = 'https://'+item
}
//convert http to https
if(item.indexOf('http://') >= 0){
allUrls[index] = item.replace('http://','https://')
}
})
//Remove all duplicates
foundUrls = [...new Set(allUrls)]
return foundUrls
}
server/helpers/UrlSafety.js
+246
View File
@@ -0,0 +1,246 @@
//
// URL normalisation and the SSRF gate.
//
// Scraped URLs come from untrusted note text and from the /pushmebaby query
// param, so every URL is validated before a socket is opened - and again after
// every redirect hop, which is the hop that actually matters (a public URL that
// 302s to 127.0.0.1 is the classic bypass).
//
// Pure except for assertFetchable, which resolves DNS.
//
const crypto = require('crypto')
const dns = require('dns').promises
const net = require('net')
const config = require('@config/scrape')
let UrlSafety = module.exports = {}
const FETCHABLE_SCHEMES = new Set(['http:', 'https:'])
// Hostname suffixes that never belong to a public site.
const BLOCKED_SUFFIXES = ['.localhost', '.local', '.internal', '.home.arpa', '.lan']
// Query params that change nothing about the page but would fragment the cache.
const TRACKING_PARAMS = [
/^utm_/i, /^fbclid$/i, /^gclid$/i, /^dclid$/i, /^msclkid$/i,
/^mc_(e|c)id$/i, /^igshid$/i, /^ref_src$/i, /^_hs(enc|mi)$/i, /^vero_/i,
]
class SafetyError extends Error {
constructor(reason, message) {
super(message || reason)
this.name = 'SafetyError'
this.reason = reason
}
}
UrlSafety.SafetyError = SafetyError
//
// Parse to a URL object, or null. Does NOT mutate anything that would change
// which resource is addressed - callers may store the result.
//
UrlSafety.parse = (raw, base) => {
if (!raw || typeof raw !== 'string') { return null }
let url = null
try {
url = new URL(raw.trim(), base || undefined)
} catch (error) {
return null
}
if (!FETCHABLE_SCHEMES.has(url.protocol)) { return null }
if (url.href.length > config.maxUrlLength) { return null }
return url
}
//
// Canonical form used ONLY for cache keys. Never store this - it deliberately
// discards tracking params and case, which would change what the user wrote.
//
UrlSafety.normalizeForKey = (raw) => {
const url = UrlSafety.parse(raw)
if (!url) { return null }
url.hash = ''
url.username = ''
url.password = ''
// Trailing dot on the host is legal DNS but splits the cache key.
if (url.hostname.endsWith('.')) {
url.hostname = url.hostname.slice(0, -1)
}
url.hostname = url.hostname.toLowerCase()
// new URL already strips the default port, but be explicit.
if ((url.protocol === 'http:' && url.port === '80') ||
(url.protocol === 'https:' && url.port === '443')) {
url.port = ''
}
for (const key of [...url.searchParams.keys()]) {
if (TRACKING_PARAMS.some(pattern => pattern.test(key))) {
url.searchParams.delete(key)
}
}
url.searchParams.sort()
if (url.pathname === '') { url.pathname = '/' }
return url
}
// sha256 of the normalised href. This is scrape_cache.url_hash - char(64),
// because a UNIQUE index on varchar(1000) utf8mb4 would be 4000 bytes and
// blow InnoDB's 3072-byte key limit.
UrlSafety.urlKey = (raw) => {
const url = UrlSafety.normalizeForKey(raw)
if (!url) { return null }
return crypto.createHash('sha256').update(url.href).digest('hex')
}
//
// True when two URLs address the same resource modulo scheme, "www." and a
// trailing slash.
//
// Load-bearing: scanTextForWebsites diffs the URLs currently in a note against
// the URLs already stored as attachments. Raw string equality would treat
// http://foo.com and https://foo.com as different resources and delete-then-
// rescrape the row on every save.
//
UrlSafety.sameResource = (a, b) => {
const left = UrlSafety.normalizeForKey(a)
const right = UrlSafety.normalizeForKey(b)
if (!left || !right) { return false }
const host = (url) => url.hostname.replace(/^www\./, '')
const path = (url) => url.pathname.replace(/\/+$/, '')
return host(left) === host(right) &&
path(left) === path(right) &&
left.search === right.search
}
UrlSafety.registrableHost = (hostname) => {
return String(hostname || '').toLowerCase().replace(/^www\./, '').replace(/\.$/, '')
}
//
// Private / non-routable address detection.
//
// Covers the ranges that matter for SSRF against this app: loopback, RFC1918,
// link-local (including the 169.254.169.254 cloud metadata endpoint), CGNAT,
// and the IPv6 equivalents.
//
UrlSafety.isPrivateIp = (address) => {
if (!address) { return true }
const version = net.isIP(address)
if (version === 0) { return true } // not an IP - caller resolves first
if (version === 4) { return isPrivateIpv4(address) }
return isPrivateIpv6(address)
}
const isPrivateIpv4 = (address) => {
const parts = address.split('.').map(Number)
if (parts.length !== 4 || parts.some(n => !Number.isInteger(n) || n < 0 || n > 255)) {
return true
}
const [a, b] = parts
if (a === 0) { return true } // 0.0.0.0/8 "this host"
if (a === 10) { return true } // RFC1918
if (a === 127) { return true } // loopback
if (a === 169 && b === 254) { return true } // link-local + metadata
if (a === 172 && b >= 16 && b <= 31) { return true } // RFC1918
if (a === 192 && b === 168) { return true } // RFC1918
if (a === 192 && b === 0) { return true } // 192.0.0/24 + 192.0.2/24
if (a === 198 && (b === 18 || b === 19)) { return true } // benchmarking
if (a === 100 && b >= 64 && b <= 127) { return true } // CGNAT
if (a === 198 && b === 51) { return true } // TEST-NET-2
if (a === 203 && b === 0) { return true } // TEST-NET-3
if (a >= 224) { return true } // multicast + reserved
return false
}
const isPrivateIpv6 = (address) => {
const lower = address.toLowerCase().replace(/^\[|\]$/g, '')
// Unwrap v4-mapped (::ffff:127.0.0.1) and NAT64 (64:ff9b::/96) before
// judging, otherwise ::ffff:127.0.0.1 reads as "some v6 address".
const mapped = lower.match(/^(?:::ffff:|64:ff9b::)(\d+\.\d+\.\d+\.\d+)$/)
if (mapped) { return isPrivateIpv4(mapped[1]) }
if (lower === '::' || lower === '::1') { return true } // unspecified, loopback
if (/^f[cd]/.test(lower)) { return true } // fc00::/7 unique-local
if (/^fe[89ab]/.test(lower)) { return true } // fe80::/10 link-local
if (/^ff/.test(lower)) { return true } // multicast
return false
}
//
// The gate. Throws SafetyError; resolves to the vetted addresses on success.
//
// Known limitation (documented, accepted): this resolves DNS and then hands the
// hostname to fetch, which resolves again. A hostile 0-TTL record can answer
// differently the second time. Closing that properly needs a custom undici
// Agent with connect.lookup - deliberately deferred, see the plan's Phase 5.
// Residual risk is blind SSRF only: no credentials are sent and no response
// body is returned to whoever planted the URL.
//
UrlSafety.assertFetchable = async (raw) => {
const url = raw instanceof URL ? raw : UrlSafety.parse(raw)
if (!url) {
throw new SafetyError('bad_url', `Not a fetchable URL: ${raw}`)
}
if (!FETCHABLE_SCHEMES.has(url.protocol)) {
throw new SafetyError('bad_scheme', `Refusing scheme ${url.protocol}`)
}
if (url.username || url.password) {
throw new SafetyError('credentials_in_url', 'Refusing URL with embedded credentials')
}
const hostname = url.hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase()
if (!hostname) {
throw new SafetyError('bad_url', 'URL has no host')
}
if (hostname === 'localhost' || BLOCKED_SUFFIXES.some(suffix => hostname.endsWith(suffix))) {
throw new SafetyError('blocked_host', `Refusing internal host ${hostname}`)
}
// Literal IP in the URL - no DNS needed.
if (net.isIP(hostname)) {
if (UrlSafety.isPrivateIp(hostname)) {
throw new SafetyError('private_ip', `Refusing non-routable address ${hostname}`)
}
return [hostname]
}
let resolved = []
try {
resolved = await dns.lookup(hostname, { all: true, verbatim: true })
} catch (error) {
throw new SafetyError('dns_failed', `Could not resolve ${hostname}: ${error.code || error.message}`)
}
if (!resolved.length) {
throw new SafetyError('dns_failed', `No addresses for ${hostname}`)
}
// Every answer must be public. One private address is enough to refuse -
// otherwise a round-robin record gets us some of the time.
for (const entry of resolved) {
if (UrlSafety.isPrivateIp(entry.address)) {
throw new SafetyError('private_ip', `${hostname} resolves to non-routable ${entry.address}`)
}
}
return resolved.map(entry => entry.address)
}
server/index.js
+65 -3
View File
@@ -345,10 +345,72 @@ app.use('/api/quick-note', quickNote)
var metricTracking = require('@routes/metrictrackingController')
app.use('/api/metric-tracking', metricTracking)
//Output running status
app.listen(ports.express, () => {
//
// Scraper startup.
//
// The application deliberately holds no DDL privileges, so it does not create
// its own schema - verifyScrapeSchema() only checks that an admin has already
// run ./migrateDatabase.sh and that the required DML grants are present.
//
// When anything is missing the scraper stays off rather than failing an INSERT
// on every note save. The listener binds either way: the API must not depend on
// link previews being available.
//
const Schema = require('@helpers/Schema')
const ScrapeQueue = require('@workers/ScrapeQueue')
Schema.verifyScrapeSchema()
.then(result => {
if (!result.ok) {
const missing = [
...result.missingObjects,
...result.missingPrivileges.map(privilege => `privilege ${privilege}`),
].join(', ')
console.log(`Scraper: DISABLED - missing ${missing}. To fix: ${result.remedy}`)
return null
}
// Say so positively. Previously a healthy scraper logged nothing, so
// "started" and "silently disabled" looked identical in pm2 logs --
// which is how a dead scraper went unnoticed on production.
console.log('Scraper: ready')
ScrapeQueue.start()
// The queue lives in memory, so a pm2 reload loses whatever was in it.
// Every enqueued URL already has an attachment row, which makes the
// work re-derivable without any extra table.
return ScrapeQueue.recoverPending()
})
.then(recovery => {
if (recovery && recovery.requeued) {
console.log(`Scraper: re-queued ${recovery.requeued} pending URL(s), retired ${recovery.retired}`)
}
})
.catch(error => {
// A scraper that cannot start must not stop the API from serving.
console.log('Scraper: startup check failed, link previews disabled this boot:', error.message)
})
.finally(() => {
//Output running status
app.listen(ports.express, () => {
console.log(`Express: Listening on port ${ports.express}!`)
})
})
})
//
// pm2 reload sends SIGINT. Stop accepting new scrape work and let in-flight
// requests abort via their own AbortControllers; rows left mid-fetch are
// recovered on the next boot.
//
const shutdown = (signal) => {
console.log(`Received ${signal}, stopping scrape queue`)
try { ScrapeQueue.stop() } catch (error) { /* already down */ }
process.exit(0)
}
process.on('SIGINT', () => shutdown('SIGINT'))
process.on('SIGTERM', () => shutdown('SIGTERM'))
//
//Error handlers
server/models/Attachment.js
+248 -239
View File
@@ -6,15 +6,30 @@ const cs = require('@helpers/CryptoString')
let Attachment = module.exports = {}
const cheerio = require('cheerio')
const rp = require('request-promise')
const request = require('request')
const fs = require('fs')
const path = require('path')
const crypto = require('crypto')
const gm = require('gm')
const tesseract = require("node-tesseract-ocr")
// Left as-is deliberately: the upload / OCR / thumbnail code below resolves
// against it, and it is correct under pm2 (cwd = server/). Not touching it
// keeps the upload path byte-identical.
const filePath = '../staticFiles/'
// Absolute equivalent, used by the scrape path only. filePath is CWD-relative,
// so under Jest (cwd = repo root) scrape downloads landed in a sibling of the
// repo rather than in staticFiles/. This resolves to the same directory as
// filePath under pm2.
const scrapeFilePath = path.join(__dirname, '..', '..', 'staticFiles') + path.sep
const config = require('@config/scrape')
const UrlSafety = require('@helpers/UrlSafety')
const HttpFetch = require('@helpers/HttpFetch')
const Realtime = require('@helpers/Realtime')
// Attachment.migrateOld
Attachment.textSearch = (userId, searchTerm) => {
@@ -263,262 +278,233 @@ Attachment.generateThumbnail = (fileName) => {
})
}
//Scans text for websites, returns all attachments
Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
return new Promise((resolve, reject) => {
//
// Diff the URLs currently in a note against the attachments already stored for
// it, soft-delete the ones that vanished, and enqueue the new ones.
//
// Returns immediately with the text already known from cache; freshly scraped
// text arrives asynchronously via the worker. Nothing here waits on the network.
//
Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
let solrAttachmentText = '' //Final searchable scrape text for note
const ScrapeQueue = require('@workers/ScrapeQueue')
const Schema = require('@helpers/Schema')
if(noteText.length == 0){ resolve(solrAttachmentText) }
// Fixed: the old version called resolve() without returning, so an empty
// note fell through and ran the DB lookup anyway.
if (!noteText || noteText.length == 0) { return Promise.resolve('') }
Attachment.urlForNote(userId, noteId).then(attachments => {
// When the scrape schema is absent the new columns do not exist, so
// creating a pending row would fail on every save. The diff and soft-delete
// below still run - they touch no new columns - so removing a link from a
// note behaves exactly as before.
const scrapeAvailable = Schema.isScrapeSchemaReady()
//Pull all the URLs out of the text
let foundUrls = SiteScrape.getCleanUrls(noteText)
let knownText = ''
return Attachment.urlForNote(userId, noteId)
.then(attachments => {
const foundUrls = SiteScrape.getCleanUrls(noteText)
const remaining = foundUrls.slice()
//Go through each saved URL, remove new URLs from saved URLs
//If a URL is not found, delete it
attachments.forEach(attachment => {
//URL already scraped, push text and continue
let urlIndex = foundUrls.indexOf( attachment.url )
if(urlIndex != -1){
solrAttachmentText += attachment.text
foundUrls.splice(urlIndex, 1) //Remove existing from set of found
// Compared on normalised keys rather than raw strings, so an
// http/https or www. variant is not treated as a different
// resource and needlessly deleted and re-scraped.
const index = remaining.findIndex(url =>
url === attachment.url || UrlSafety.sameResource(url, attachment.url))
if (index !== -1) {
knownText += attachment.text || ''
remaining.splice(index, 1)
} else {
//If existing attachment is not found in note, remove it
// Unchanged behaviour: link attachments are hidden, not deleted.
Attachment.delete(userId, attachment.id, true)
}
})
//No newly scraped URLs, resolve with looked up attachment text
if(foundUrls == null || foundUrls.length == 0){
return resolve(solrAttachmentText)
}
if (!remaining.length) { return knownText }
//Process the remaining URLs into attachments
Attachment.scrapeUrlsCreateAttachments(userId, noteId, foundUrls).then( freshlyScrapedText => {
// No schema, no previews. Return what is already cached rather than
// attempting inserts that cannot succeed.
if (!scrapeAvailable) { return knownText }
//Once everything is done being scraped, emit new attachment events
SocketIo.to(`user:${userId}`).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(`user:${userId}`).emit('update_note_attachments')
solrAttachmentText += freshlyScrapedText
resolve(solrAttachmentText)
// Create a shell row per new URL, then hand off to the worker.
return Promise.all(remaining.map(url =>
Attachment.createPendingUrlAttachment(userId, noteId, url)
.then(attachmentId => ({ userId, noteId, attachmentId, url }))
.catch(error => {
console.log('[scrape] could not create attachment row', url, error.message)
return null
})
.catch(console.log)
))
.then(jobs => {
const valid = jobs.filter(Boolean)
// Fire and forget - the note save must not wait on this.
ScrapeQueue.enqueueUrls(valid)
.catch(error => console.log('[scrape] enqueue failed', error.message))
return knownText
})
})
// The old version had no catch here at all.
.catch(error => {
console.log('[scrape] scanTextForWebsites failed', error.message)
return knownText
})
}
//Return scraped text from each URL
//
// Create the shell attachment row for a URL and return its id.
//
// Returns the id BEFORE any network work begins. The old code armed a 5s
// setTimeout calling request.cancel() synchronously, before this INSERT had
// resolved - so a slow INSERT meant the timer fired against a null reference
// and threw inside a bare timer callback, taking the process down.
//
Attachment.createPendingUrlAttachment = (userId, noteId, url) => {
const created = Math.round((+new Date) / 1000)
return db.promise()
.query(
`INSERT INTO attachment
(note_id, user_id, attachment_type, text, url, last_indexed, file_location, scrape_state, enqueued_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[noteId, userId, 1, url, url, created, null, 0, created]
)
.then(([result]) => result.insertId)
}
//
// Scrape a set of URLs for a note and resolve with their combined text.
//
// Kept for the bookmarklet path. The old implementation resolved only when a
// completion counter reached foundUrls.length, so a single rejection meant the
// promise never settled and the caller hung forever. allSettled inside the
// worker removes the counter entirely.
//
Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
return new Promise((resolve, reject) => {
if(foundUrls == null || foundUrls.length == 0){
return resolve('')
const ScrapeQueue = require('@workers/ScrapeQueue')
const Schema = require('@helpers/Schema')
if (!foundUrls || !foundUrls.length) { return Promise.resolve('') }
if (!Schema.isScrapeSchemaReady()) { return Promise.resolve('') }
return Promise.all(foundUrls.map(url =>
Attachment.createPendingUrlAttachment(userId, noteId, url)
.then(attachmentId => ({ userId, noteId, attachmentId, url }))
.catch(error => {
console.log('[scrape] could not create attachment row', url, error.message)
return null
})
))
.then(jobs => ScrapeQueue.enqueueUrls(jobs.filter(Boolean)))
.then(texts => texts.join(''))
}
//
// Download and store a preview image.
//
// The old downloadFileFromUrl had no timeout, no size limit, no User-Agent and
// no status check: it piped whatever came back straight into a write stream, so
// a 404 HTML error page was written to disk and handed to gm. It also wrote the
// file before validating anything, which is why staticFiles/ accumulated
// zero-byte orphans.
//
Attachment.downloadThumbnail = (imageUrl, pageUrl) => {
if (!imageUrl) { return Promise.resolve(null) }
// crypto.randomBytes rather than two concatenated Math.random() calls.
const fileName = crypto.randomBytes(16).toString('hex') + '_scrape'
const fullPath = scrapeFilePath + fileName
const thumbPath = scrapeFilePath + 'thumb_' + fileName
const tempPath = fullPath + '.tmp'
return HttpFetch.fetchBinary(imageUrl, {
pageUrl,
timeoutMs: config.imageTimeoutMs,
maxBytes: config.maxImageBytes,
})
.then(result => {
// Status and content-type were both gated inside fetchBinary, so by
// here we are holding bytes that really are an image.
if (!result.buffer || !result.buffer.length) { return null }
if (!magicBytesMatch(result.buffer, result.mime)) {
console.log('[scrape] image bytes do not match declared type', result.mime, imageUrl)
return null
}
console.log('About to scrape')
console.log(foundUrls)
let processedCount = 0
let scrapedText = ''
//Process each URL passd to function, a DB entry will be created for each scrape
foundUrls.forEach(url => {
Attachment.processUrl(userId, noteId, url).then( freshlyScrapedText => {
scrapedText += freshlyScrapedText
processedCount ++
//All URLs have been scraped, return data
if(processedCount == foundUrls.length){
console.log('All urls scraped')
return resolve(scrapedText)
}
// Write to a temp path, then rename, so a failure cannot leave a
// half-written file behind for the client to serve.
return fs.promises.writeFile(tempPath, result.buffer)
.then(() => fs.promises.rename(tempPath, fullPath))
.then(() => resizeToThumbnail(fullPath, thumbPath))
.then(() => fileName)
.catch(error => {
console.log('[scrape] could not store image', imageUrl, error.message)
return cleanupFiles([tempPath, fullPath, thumbPath]).then(() => null)
})
})
.catch(error => {
console.log('Site Scrape error', error)
})
})
console.log('[scrape] image fetch failed', imageUrl, error.kind || error.message)
return null
})
}
Attachment.downloadFileFromUrl = (url) => {
return new Promise((resolve, reject) => {
if(!url){
return resolve(null)
}
const random = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
let extension = ''
let fileName = random+'_scrape'
let thumbPath = 'thumb_'+fileName
console.log('Scraping image url', url)
console.log('Getting ready to scrape ', url)
request(url)
.on('error', error => {
console.log(error)
resolve(null)
})
.on('response', res => {
console.log(res.statusCode)
console.log(res.headers['content-type'])
//Get mime type from header content type
// extension = '.'+String(res.headers['content-type']).split('/').pop()
})
.pipe(fs.createWriteStream(filePath+thumbPath))
.on('close', () => {
//resize image if its real big
gm(filePath+thumbPath)
//
// Resize via gm. Degrades rather than failing: gm (and tesseract) are not in
// shell.nix buildInputs, so the binary is absent in dev - the old code returned
// null here and left the downloaded file orphaned on disk. Copying the
// full-size image to the thumbnail path keeps the preview working.
//
const resizeToThumbnail = (fullPath, thumbPath) => {
return new Promise(resolve => {
gm(fullPath)
.resize(550) //Resize to width of 550 px
.quality(85) //compression level 0 - 100 (best)
.write(filePath+thumbPath, function (err) {
if(err){
console.log(err)
return resolve(null)
}
console.log('Saved Image')
return resolve(fileName)
})
.write(thumbPath, error => {
if (!error) { return resolve(true) }
console.log('[scrape] gm unavailable or failed, using full size as thumbnail')
fs.promises.copyFile(fullPath, thumbPath)
.then(() => resolve(true))
.catch(() => resolve(false))
})
})
}
Attachment.processUrl = (userId, noteId, url) => {
const cleanupFiles = (paths) => {
return Promise.all(paths.map(target =>
fs.promises.unlink(target).catch(() => null)
))
}
const scrapeTime = 5*1000;
//
// Sanity-check the bytes against the declared content type. A server can claim
// image/png and send anything.
//
const IMAGE_MAGIC = [
{ mime: 'image/png', bytes: [0x89, 0x50, 0x4e, 0x47] },
{ mime: 'image/jpeg', bytes: [0xff, 0xd8, 0xff] },
{ mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] },
{ mime: 'image/bmp', bytes: [0x42, 0x4d] },
]
return new Promise((resolve, reject) => {
const magicBytesMatch = (buffer, mime) => {
const expected = IMAGE_MAGIC.find(entry => entry.mime === mime)
// webp/avif/ico and friends have container-dependent headers - accept them
// on the content-type alone rather than guessing wrong.
if (!expected) { return true }
if (buffer.length < expected.bytes.length) { return false }
const options = {
uri: url,
simple: true,
timeout: scrapeTime,
headers: {
'User-Agent':'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' //Simulate google headers
},
transform: function (body) {
return cheerio.load(body);
}
}
let requestTimeout = null
let thumbnail = null
let request = null
let created = Math.round((+new Date)/1000)
let insertedId = null
//Create a shell attachment for each URL, put in processing state
db.promise()
.query(`INSERT INTO attachment
(note_id, user_id, attachment_type, text, url, last_indexed, file_location)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
[noteId, userId, 1, url, url, created, null])
.then((rows, fields) => {
//Set two bigger variables then return request for processing
request = rp(options)
insertedId = rows[0].insertId
return request
})
.then($ => {
//Clear timeout that would end this function
clearTimeout(requestTimeout)
// let header = $('h1').text().replace(removeWhitespace, " ")
// desiredSearchText += header + "\n"
const pageTitle = SiteScrape.getTitle($)
const hostname = SiteScrape.getHostName(url)
const thumbnail = SiteScrape.getDisplayImage($, url)
const keywords = SiteScrape.getKeywords($)
var desiredSearchText = ''
desiredSearchText += pageTitle
if(keywords){
desiredSearchText += "\n " + keywords
}
console.log('Results from site scrape-------------')
console.log({
pageTitle,
hostname,
thumbnail,
keywords
})
// throw new Error('Ending this function early.')
// console.log('TexT Scraped')
// console.log(desiredSearchText)
created = Math.round((+new Date)/1000)
//Scrape URL for thumbnail - take filename and save in attachment
Attachment.downloadFileFromUrl(thumbnail)
.then(thumbnailFilename => {
//Update text and thumbnail filename
created = Math.round((+new Date)/1000)
db.promise()
.query(`UPDATE attachment SET
text = ?,
last_indexed = ?,
file_location = ?
WHERE id = ?
`, [desiredSearchText, created, thumbnailFilename, insertedId])
.then((rows, fields) => {
resolve(desiredSearchText) //Return found text
})
.catch(console.log)
//Create attachment in DB with scrape text and provided data
// db.promise()
// .query(`INSERT INTO attachment
// (note_id, user_id, attachment_type, text, url, last_indexed, file_location)
// VALUES (?, ?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created, thumbnailFilename])
// .then((rows, fields) => {
// resolve(desiredSearchText) //Return found text
// })
// .catch(console.log)
})
})
.catch(error => {
console.log('Scrape pooped out')
console.log('Issue with scrape', error.statusCode)
clearTimeout(requestTimeout)
return resolve('No site text')
})
requestTimeout = setTimeout( () => {
console.log('Cancel the request, its taking to long.')
request.cancel()
return resolve('Request Timeout')
}, scrapeTime )
})
return expected.bytes.every((byte, index) => buffer[index] === byte)
}
Attachment.generatePushKey = (userId) => {
@@ -576,36 +562,59 @@ Attachment.getPushkeyBookmarklet = (userId) => {
})
}
Attachment.pushUrl = (pushkey,url) => {
return new Promise((resolve, reject) => {
//
// Bookmarklet entry point: GET /api/public/pushmebaby?pushkey=...&url=...
//
// The URL arrives straight off a query string, so it is validated here - the
// old code passed it through completely unvalidated, bypassing even the https
// normalisation that note text goes through.
//
// Response timing is preserved: the bookmarklet still answers after the scrape,
// but now with a hard cap instead of the previous unbounded hang.
//
Attachment.pushUrl = (pushkey, url) => {
const ScrapeQueue = require('@workers/ScrapeQueue')
const Schema = require('@helpers/Schema')
// Preserve the bookmarklet contract - resolve truthy, leaking nothing -
// rather than failing an insert against columns that do not exist.
if (!Schema.isScrapeSchemaReady()) { return Promise.resolve(true) }
let userId = null
pushkey = pushkey.replace(/ /g, '+')
pushkey = String(pushkey || '').replace(/ /g, '+')
db.promise()
return db.promise()
.query("SELECT id FROM user WHERE pushkey = ? LIMIT 1", [pushkey])
.then((rows, fields) => {
.then(([rows]) => {
if(rows[0].length == 0){
return resolve(true)
}
// Unchanged: an unknown pushkey resolves truthy and leaks nothing
// about whether the key exists.
if (rows.length == 0) { return null }
userId = rows[0][0].id
return Attachment.scrapeUrlsCreateAttachments(userId, null, [url])
userId = rows[0].id
return UrlSafety.assertFetchable(url)
.then(() => Attachment.createPendingUrlAttachment(userId, null, url))
.then(attachmentId => ScrapeQueue.awaitUrl(
ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url }),
config.totalBudgetMs
))
.catch(error => {
console.log('[scrape] pushUrl refused', url, error.reason || error.message)
return null
})
})
.then(() => {
if(typeof SocketIo != 'undefined'){
//Once everything is done being scraped, emit new attachment events
SocketIo.to(`user:${userId}`).emit('update_counts')
// Tell user to update attachments with scraped text
SocketIo.to(`user:${userId}`).emit('update_note_attachments')
if (userId) {
Realtime.emitToUser(userId, 'update_counts')
Realtime.emitToUser(userId, 'update_note_attachments')
}
return resolve(true)
return true
})
.catch(console.log)
.catch(error => {
console.log('[scrape] pushUrl failed', error.message)
return true
})
}
server/models/Note.js
+8 -5
View File
@@ -4,6 +4,7 @@ let Note = module.exports = {}
let Tags = require('@models/Tag')
let Attachment = require('@models/Attachment')
let Realtime = require('@helpers/Realtime')
let ShareNote = require('@models/ShareNote')
let ProcessText = require('@helpers/ProcessText')
@@ -12,7 +13,6 @@ const DiffMatchPatch = require('@helpers/DiffMatchPatch')
const crypto = require('crypto')
const cs = require('@helpers/CryptoString')
const rp = require('request-promise');
const fs = require('fs')
const gm = require('gm')
@@ -513,12 +513,15 @@ Note.update = (userId, noteId, noteText, noteTitle, color, pinned, archived, has
})
.then((rows, fields) => {
if(typeof SocketIo != 'undefined'){
SocketIo.to(`user:${userId}`).emit('new_note_text_saved', {noteId, hash})
// The old `if(typeof SocketIo != 'undefined')` guard did not just
// skip the emit - it skipped the scrape too, so link scraping was
// silently disabled under Jest and could never be tested. Realtime
// no-ops safely without a socket server, so both now always run.
Realtime.emitToUser(userId, 'new_note_text_saved', {noteId, hash})
//Async attachment reindex
Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
}
Attachment.scanTextForWebsites(userId, noteId, noteText)
.catch(error => console.log('scanTextForWebsites failed', error.message))
//Send back updated response
resolve(rows[0])
server/models/ScrapeCache.js
+224
View File
@@ -0,0 +1,224 @@
//
// Read-through cache for scraped URL metadata, keyed by a hash of the
// normalised URL and shared across users and notes.
//
// A fresh hit means zero network I/O. A stale hit means a conditional request
// (If-None-Match / If-Modified-Since), so an unchanged page costs one 304
// rather than a full page load. Failures are backed off so a dead link is not
// refetched on every note save.
//
// Stores only public page metadata. Note content never reaches this table.
//
const db = require('@config/database')
const UrlSafety = require('@helpers/UrlSafety')
let ScrapeCache = module.exports = {}
const SUCCESS_TTL_SECONDS = 30 * 24 * 60 * 60 // 30 days
// Widening backoff on repeated failure, then give up and stay failed.
const FAILURE_BACKOFF_SECONDS = [60 * 60, 6 * 60 * 60, 24 * 60 * 60]
const MAX_FAILURES = FAILURE_BACKOFF_SECONDS.length
ScrapeCache.SUCCESS_TTL_SECONDS = SUCCESS_TTL_SECONDS
ScrapeCache.MAX_FAILURES = MAX_FAILURES
const now = () => Math.round(Date.now() / 1000)
const truncate = (value, length) => {
if (value == null) { return null }
const text = String(value)
return text.length > length ? text.substring(0, length) : text
}
ScrapeCache.get = async (urlOrKey) => {
const key = /^[0-9a-f]{64}$/.test(urlOrKey) ? urlOrKey : UrlSafety.urlKey(urlOrKey)
if (!key) { return null }
const [rows] = await db.promise().query('SELECT * FROM scrape_cache WHERE url_hash = ? LIMIT 1', [key])
return rows.length ? rows[0] : null
}
// Fresh = inside its TTL and successfully fetched.
ScrapeCache.isFresh = (row, at) => {
if (!row || !row.expires_at) { return false }
return row.expires_at > (at || now())
}
ScrapeCache.isUsable = (row) => {
return !!(row && row.fetch_state === 'ok' && row.search_text != null)
}
// Has this URL failed enough times that we should stop trying?
ScrapeCache.isExhausted = (row) => {
return !!(row && row.fetch_state === 'failed' && row.failure_count >= MAX_FAILURES)
}
//
// Conditional-request headers for a stale-but-successful row. Null when there
// is nothing to revalidate against.
//
ScrapeCache.conditionalFor = (row) => {
if (!row || row.fetch_state !== 'ok') { return null }
if (!row.etag && !row.last_modified) { return null }
return { etag: row.etag || null, lastModified: row.last_modified || null }
}
//
// Reserve a row before fetching, so a concurrent save for the same URL finds
// it rather than starting a second fetch.
//
ScrapeCache.markPending = async (url) => {
const key = UrlSafety.urlKey(url)
if (!key) { return null }
await db.promise().query(
`INSERT INTO scrape_cache (url_hash, url, fetch_state, fetched_at)
VALUES (?, ?, 'pending', ?)
ON DUPLICATE KEY UPDATE fetch_state = IF(fetch_state = 'ok', fetch_state, 'pending')`,
[key, truncate(url, 1000), now()]
)
return key
}
//
// Store a successful scrape.
//
ScrapeCache.putSuccess = async (url, result) => {
const key = UrlSafety.urlKey(url)
if (!key) { return null }
const at = now()
const fields = {
url: truncate(url, 1000),
canonical_url: truncate(result.canonicalUrl, 1000),
status: result.status || 200,
title: truncate(result.title, 512),
site_name: truncate(result.siteName, 255),
description: result.description || null,
keywords: truncate(result.keywords, 512),
search_text: result.searchText || null,
image_url: truncate(result.imageUrl, 1000),
file_location: truncate(result.fileLocation, 500),
etag: truncate(result.etag, 255),
last_modified: truncate(result.lastModified, 128),
content_type: truncate(result.contentType, 128),
fetch_state: 'ok',
failure_count: 0,
last_error: null,
fetched_at: at,
expires_at: at + SUCCESS_TTL_SECONDS,
}
const columns = Object.keys(fields)
const values = columns.map(column => fields[column])
await db.promise().query(
`INSERT INTO scrape_cache (url_hash, ${columns.join(', ')})
VALUES (?, ${columns.map(() => '?').join(', ')})
ON DUPLICATE KEY UPDATE ${columns.map(column => `${column} = VALUES(${column})`).join(', ')}`,
[key, ...values]
)
return key
}
//
// A 304. The page is unchanged, so only the freshness window moves.
//
// Deliberately does NOT touch attachment.last_indexed - Attachment.search
// orders by it, so revalidating would silently reshuffle the user's list.
//
ScrapeCache.touchNotModified = async (url) => {
const key = UrlSafety.urlKey(url)
if (!key) { return null }
const at = now()
await db.promise().query(
`UPDATE scrape_cache
SET fetched_at = ?, expires_at = ?, fetch_state = 'ok', failure_count = 0, last_error = NULL
WHERE url_hash = ?`,
[at, at + SUCCESS_TTL_SECONDS, key]
)
return key
}
//
// Record a failure with a widening retry window.
//
// `blocked` (robots or the safety gate) is terminal: retrying will not change
// the answer, so it is parked for the full success TTL instead of backing off.
//
ScrapeCache.recordFailure = async (url, error) => {
const key = UrlSafety.urlKey(url)
if (!key) { return null }
const existing = await ScrapeCache.get(key)
const failureCount = (existing ? existing.failure_count : 0) + 1
const kind = (error && error.kind) || 'unknown'
const terminal = kind === 'blocked' || kind === 'robots_disallowed'
let ttl = null
if (terminal) {
ttl = SUCCESS_TTL_SECONDS
} else if (error && error.retryAfterMs) {
ttl = Math.round(error.retryAfterMs / 1000)
} else {
ttl = FAILURE_BACKOFF_SECONDS[Math.min(failureCount, MAX_FAILURES) - 1]
}
const at = now()
const state = terminal ? 'blocked' : 'failed'
await db.promise().query(
`INSERT INTO scrape_cache
(url_hash, url, status, fetch_state, failure_count, last_error, fetched_at, expires_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
status = VALUES(status),
fetch_state = VALUES(fetch_state),
failure_count = VALUES(failure_count),
last_error = VALUES(last_error),
fetched_at = VALUES(fetched_at),
expires_at = VALUES(expires_at)`,
[
key,
truncate(url, 1000),
(error && error.status) || 0,
state,
failureCount,
truncate(`${kind}: ${(error && error.message) || ''}`, 255),
at,
at + ttl,
]
)
return key
}
//
// Project a cache row onto the shape Attachment rows want.
//
ScrapeCache.toAttachmentFields = (row) => {
if (!row) { return null }
return {
text: row.search_text || '',
fileLocation: row.file_location || null,
url: row.canonical_url || row.url,
}
}
ScrapeCache.purgeExpired = async (limit = 500) => {
const [result] = await db.promise().query(
`DELETE FROM scrape_cache
WHERE expires_at IS NOT NULL AND expires_at < ? AND fetch_state <> 'ok'
LIMIT ?`,
[now(), limit]
)
return result.affectedRows || 0
}
server/models/User.js
+358 -341
View File
@@ -1,208 +1,210 @@
const crypto = require('crypto')
const crypto = require("crypto");
const Note = require('@models/Note')
const Note = require("@models/Note");
const db = require('@config/database')
const Auth = require('@helpers/Auth')
const cs = require('@helpers/CryptoString')
const speakeasy = require('speakeasy')
const db = require("@config/database");
const Auth = require("@helpers/Auth");
const cs = require("@helpers/CryptoString");
const speakeasy = require("speakeasy");
let User = module.exports = {}
let User = (module.exports = {});
const version = '3.8.0'
const version = "4.0.0";
// 3.7.3 - diff/patch update
//Login a user, if that user does not exist create them
//Issues login token
User.login = (username, password, authToken = null) => {
return new Promise((resolve, reject) => {
const lowerName = username.toLowerCase()
const lowerName = username.toLowerCase();
let statusObject = {
success: false,
token: null,
userId: null,
verificationRequired: false,
message: 'Incorrect Username or Password'
}
message: "Incorrect Username or Password",
};
db.promise()
.query('SELECT * FROM user WHERE username = ? LIMIT 1', [lowerName])
.query("SELECT * FROM user WHERE username = ? LIMIT 1", [lowerName])
.then((rows, fields) => {
//
// Login User
//
if(rows[0].length == 1){
if (rows[0].length == 1) {
//Pull out user data from database results
const lookedUpUser = rows[0][0]
const lookedUpUser = rows[0][0];
//Verify Token if set
const tokenValidates = speakeasy.totp.verify({
'secret': lookedUpUser['two_fa_secret'],
'encoding': 'base32',
'token': authToken,
'window': 2
})
secret: lookedUpUser["two_fa_secret"],
encoding: "base32",
token: authToken,
window: 2,
});
if(lookedUpUser.two_fa_enabled == 1 && !authToken){
if (lookedUpUser.two_fa_enabled == 1 && !authToken) {
statusObject["verificationRequired"] = true;
statusObject["message"] = "2FA authentication required.";
statusObject['verificationRequired'] = true
statusObject['message'] = '2FA authentication required.'
return resolve(statusObject)
return resolve(statusObject);
}
if(lookedUpUser.two_fa_enabled == 1 && !tokenValidates){
if (lookedUpUser.two_fa_enabled == 1 && !tokenValidates) {
statusObject["verificationRequired"] = true;
statusObject["message"] = "Invalid Authorization Token.";
statusObject['verificationRequired'] = true
statusObject['message'] = 'Invalid Authorization Token.'
return resolve(statusObject)
return resolve(statusObject);
}
if(lookedUpUser.two_fa_enabled == 0 || (lookedUpUser.two_fa_enabled == 1 && tokenValidates) ){
if (
lookedUpUser.two_fa_enabled == 0 ||
(lookedUpUser.two_fa_enabled == 1 && tokenValidates)
) {
//hash the password and check for a match
const salt = Buffer.from(lookedUpUser.salt, 'binary')
crypto.pbkdf2(password, salt, lookedUpUser.iterations, 512, 'sha512', function(err, delivered_key){
if(delivered_key.toString('hex') === lookedUpUser.password){
const salt = Buffer.from(lookedUpUser.salt, "binary");
crypto.pbkdf2(
password,
salt,
lookedUpUser.iterations,
512,
"sha512",
function (err, delivered_key) {
if (delivered_key.toString("hex") === lookedUpUser.password) {
User.generateMasterKey(lookedUpUser.id, password)
.then( result => User.getMasterKey(lookedUpUser.id, password))
.then(masterKey => {
User.generateKeypair(lookedUpUser.id, masterKey)
.then(({publicKey, privateKey}) => {
.then((result) =>
User.getMasterKey(lookedUpUser.id, password),
)
.then((masterKey) => {
User.generateKeypair(lookedUpUser.id, masterKey).then(
({ publicKey, privateKey }) => {
//Passback a json web token
Auth.createToken(lookedUpUser.id, masterKey)
.then(token => {
statusObject['token'] = token
statusObject['userId'] = lookedUpUser.id
statusObject['success'] = true
return resolve(statusObject)
})
})
})
Auth.createToken(lookedUpUser.id, masterKey).then(
(token) => {
statusObject["token"] = token;
statusObject["userId"] = lookedUpUser.id;
statusObject["success"] = true;
return resolve(statusObject);
},
);
},
);
});
} else {
return resolve(statusObject)
return resolve(statusObject);
}
})
},
);
}
} else {
//If user is not found, say two factor authentication is required
statusObject['verificationRequired'] = true
statusObject['message'] = '2FA authentication required.'
statusObject["verificationRequired"] = true;
statusObject["message"] = "2FA authentication required.";
//Show fake auth token message
if(authToken){
statusObject['message'] = 'Invalid Authorization Token.'
if (authToken) {
statusObject["message"] = "Invalid Authorization Token.";
}
return resolve(statusObject)
return resolve(statusObject);
}
})
.catch(console.log)
})
}
.catch(console.log);
});
};
//Create user account
//Issues login token
User.register = (username, password) => {
//For some reason, username won't get into the promise. But password will @TODO figure this out
const lowerName = username.toLowerCase().trim()
const lowerName = username.toLowerCase().trim();
return new Promise((resolve, reject) => {
db.promise()
.query('SELECT * FROM user WHERE username = ? LIMIT 1', [lowerName])
.query("SELECT * FROM user WHERE username = ? LIMIT 1", [lowerName])
.then((rows, fields) => {
if(rows[0].length === 0){ //No users returned, create new one. Start with hashing password
if (rows[0].length === 0) {
//No users returned, create new one. Start with hashing password
//Params for hash function
let shasum = crypto.createHash('sha512') //Prepare Hash
const ran = parseInt(Date.now()) //Get current time in miliseconds
const semiRandomInt = Math.floor(Math.random()*11) //Grab a random number
const otherRandomInt = (ran*semiRandomInt+ran)*semiRandomInt-ran //Mix things up a bit
shasum.update(''+otherRandomInt) //Update Hasd
let shasum = crypto.createHash("sha512"); //Prepare Hash
const ran = parseInt(Date.now()); //Get current time in miliseconds
const semiRandomInt = Math.floor(Math.random() * 11); //Grab a random number
const otherRandomInt =
(ran * semiRandomInt + ran) * semiRandomInt - ran; //Mix things up a bit
shasum.update("" + otherRandomInt); //Update Hasd
const saltString = shasum.digest('hex')
const salt = Buffer.from(saltString, 'binary') //Generate Salt hash
const iterations = 25000
crypto.pbkdf2(password, salt, iterations, 512, 'sha512', function(err, delivered_key) {
const saltString = shasum.digest("hex");
const salt = Buffer.from(saltString, "binary"); //Generate Salt hash
const iterations = 25000;
crypto.pbkdf2(
password,
salt,
iterations,
512,
"sha512",
function (err, delivered_key) {
//Create new user object with freshly salted password
var currentDate = new Date().toISOString().slice(0, 19).replace('T', ' ');
var currentDate = new Date()
.toISOString()
.slice(0, 19)
.replace("T", " ");
var new_user = {
username: lowerName,
password: delivered_key.toString('hex'),
password: delivered_key.toString("hex"),
salt: salt,
iterations: iterations,
last_login: currentDate,
created: currentDate
created: currentDate,
};
let userId = null
let newMasterKey = null
let userId = null;
let newMasterKey = null;
db.promise()
.query('INSERT INTO user SET ?', new_user)
.query("INSERT INTO user SET ?", new_user)
.then((rows, fields) => {
userId = rows[0].insertId
return User.generateMasterKey(userId, password)
userId = rows[0].insertId;
return User.generateMasterKey(userId, password);
})
.then( result => {
return User.getMasterKey(userId, password)
.then((result) => {
return User.getMasterKey(userId, password);
})
.then(masterKey => {
newMasterKey = masterKey
return User.generateKeypair(userId, newMasterKey)
.then((masterKey) => {
newMasterKey = masterKey;
return User.generateKeypair(userId, newMasterKey);
})
.then(({publicKey, privateKey}) => {
return Auth.createToken(userId, newMasterKey)
.then(({ publicKey, privateKey }) => {
return Auth.createToken(userId, newMasterKey);
})
.then(token => {
return resolve({token, userId})
})
.catch(console.log)
.then((token) => {
return resolve({ token, userId });
})
.catch(console.log);
},
);
} else {
return reject('Username already in use.')
}//END user create
return reject("Username already in use.");
} //END user create
})
.catch(console.log)
})
}
.catch(console.log);
});
};
//Counts notes, pinned notes, archived notes, shared notes, unread notes, total files and types
User.getCounts = (userId, extendedOptions) => {
return new Promise((resolve, reject) => {
let countTotals = {
tags: {}
}
tags: {},
};
// const userHash = cs.hash(String(userId)).toString('base64')
db.promise().query(
db.promise()
.query(
`SELECT
SUM(archived = 1 && share_user_id IS NULL && trashed = 0) AS archivedNotes,
SUM(trashed = 1) AS trashedNotes,
@@ -211,22 +213,23 @@ User.getCounts = (userId, extendedOptions) => {
SUM(share_user_id != ? && trashed = 0) AS sharedToNotes
FROM note
LEFT JOIN note_raw_text ON (note_raw_text.id = note.note_raw_text_id)
WHERE user_id = ?`, [userId, userId, userId, userId])
.then( (rows, fields) => {
Object.assign(countTotals, rows[0][0]) //combine results
WHERE user_id = ?`,
[userId, userId, userId, userId],
)
.then((rows, fields) => {
Object.assign(countTotals, rows[0][0]); //combine results
//
// @TODO - Figured out if this is useful
// We want, notes shared with user and note user has shared
//
return db.promise().query(
`SELECT count(id) AS sharedFromNotes
FROM note WHERE shared = 2 AND user_id = ? AND trashed = 0`, [userId]
)
FROM note WHERE shared = 2 AND user_id = ? AND trashed = 0`,
[userId],
);
})
.then( (rows, fields) => {
Object.assign(countTotals, rows[0][0]) //combine results
.then((rows, fields) => {
Object.assign(countTotals, rows[0][0]); //combine results
return db.promise().query(
`SELECT
@@ -235,20 +238,26 @@ User.getCounts = (userId, extendedOptions) => {
COUNT(id) as totalFiles
FROM attachment WHERE visible = 1
AND user_id = ?
`, [userId]
)
}).then( (rows, fields) => {
`,
[userId],
);
})
.then((rows, fields) => {
Object.assign(countTotals, rows[0][0]); //combine results
Object.assign(countTotals, rows[0][0]) //combine results
return db.promise().query('SELECT id AS quickNote FROM note WHERE quick_note = 1 AND user_id = ?', [userId])
}).then( (rows, fields) => {
Object.assign(countTotals, rows[0][0]) //combine results
return db
.promise()
.query(
"SELECT id AS quickNote FROM note WHERE quick_note = 1 AND user_id = ?",
[userId],
);
})
.then((rows, fields) => {
Object.assign(countTotals, rows[0][0]); //combine results
//Count usages of user tags, sort by most popular
return db.promise().query(`
return db.promise().query(
`
SELECT
tag.text, COUNT(tag_id) AS uses, tag.id
FROM note_tag
@@ -257,337 +266,345 @@ User.getCounts = (userId, extendedOptions) => {
GROUP BY tag_id
ORDER BY uses DESC
LIMIT 16
`, [userId])
}).then( (rows, fields) => {
//Convert everything to an int or 0
Object.keys(countTotals).forEach( key => {
const count = parseInt(countTotals[key])
countTotals[key] = count ? count : 0
`,
[userId],
);
})
.then((rows, fields) => {
//Convert everything to an int or 0
Object.keys(countTotals).forEach((key) => {
const count = parseInt(countTotals[key]);
countTotals[key] = count ? count : 0;
});
//Build out tags object
let tagsObject = {}
rows[0].forEach(tagRow => {
tagsObject[tagRow['text']] = {'id':tagRow.id, 'uses':tagRow.uses}
})
let tagsObject = {};
rows[0].forEach((tagRow) => {
tagsObject[tagRow["text"]] = { id: tagRow.id, uses: tagRow.uses };
});
//Assign after counts are updated
countTotals['tags'] = tagsObject
countTotals["tags"] = tagsObject;
countTotals['currentVersion'] = version
countTotals["currentVersion"] = version;
// Allow for extended options set on page load
if(extendedOptions){
db.promise().query(
if (extendedOptions) {
db.promise()
.query(
`SELECT updated FROM note
JOIN note_raw_text ON note_raw_text.id = note.note_raw_text_id
WHERE note.quick_note = 2
AND user_id = ?`, [userId])
.then( (rows, fields) => {
if(rows[0][0] && rows[0][0].updated){
const lastOpened = rows[0][0].updated
const timeDiff = Math.round(((+new Date) - (lastOpened))/1000)
const hoursInSeconds = (12 * 60 * 60) //12 hours
AND user_id = ?`,
[userId],
)
.then((rows, fields) => {
if (rows[0][0] && rows[0][0].updated) {
const lastOpened = rows[0][0].updated;
const timeDiff = Math.round((+new Date() - lastOpened) / 1000);
const hoursInSeconds = 12 * 60 * 60; //12 hours
// Show metric tracking button if its been 12 hours since last entry
if(lastOpened && timeDiff > hoursInSeconds){
countTotals['showTrackMetricsButton'] = true
if (lastOpened && timeDiff > hoursInSeconds) {
countTotals["showTrackMetricsButton"] = true;
}
}
resolve(countTotals)
})
resolve(countTotals);
});
} else {
resolve(countTotals)
resolve(countTotals);
}
})
})
}
});
});
};
//Log out user by deleting login token for that active session
User.logout = (sessionId) => {
console.log('Terminate Session -> ', sessionId)
return db.promise().query('DELETE FROM user_active_session WHERE (session_id = ?)', [sessionId])
}
console.log("Terminate Session -> ", sessionId);
return db
.promise()
.query("DELETE FROM user_active_session WHERE (session_id = ?)", [
sessionId,
]);
};
User.generateMasterKey = (userId, password) => {
return new Promise((resolve, reject) => {
if(!userId || !password){
reject('Need userId and password to generate key')
if (!userId || !password) {
reject("Need userId and password to generate key");
}
db.promise()
.query('SELECT count(id) as total FROM user_key WHERE user_id = ?', [userId])
.query("SELECT count(id) as total FROM user_key WHERE user_id = ?", [
userId,
])
.then((rows, fields) => {
//Entry already exists, you good.
if(rows[0][0]['total'] > 0){
return resolve(true)
if (rows[0][0]["total"] > 0) {
return resolve(true);
// throw new Error('User Encryption key already exists')
} else {
// Generate user key, its big and random
const masterPassword = cs.createSmallSalt()
const masterPassword = cs.createSmallSalt();
//Generate a salt because it wants it
const salt = cs.createSmallSalt()
const salt = cs.createSmallSalt();
// Encrypt master password
const encryptedMasterPassword = cs.encrypt(password, salt, masterPassword)
const encryptedMasterPassword = cs.encrypt(
password,
salt,
masterPassword,
);
const created = Math.round((+new Date)/1000)
const created = Math.round(+new Date() / 1000);
db.promise()
.query(
'INSERT INTO user_key (`user_id`, `salt`, `key`, `created`) VALUES (?, ?, ?, ?);',
[userId, salt, encryptedMasterPassword, created]
"INSERT INTO user_key (`user_id`, `salt`, `key`, `created`) VALUES (?, ?, ?, ?);",
[userId, salt, encryptedMasterPassword, created],
)
.then(results => {
return resolve(true)
})
.then((results) => {
return resolve(true);
});
}
})
.catch(error => {
console.log('Create Master Password Error')
console.log(error)
})
})
}
.catch((error) => {
console.log("Create Master Password Error");
console.log(error);
});
});
};
User.getMasterKey = (userId, password) => {
return new Promise((resolve, reject) => {
if(!userId || !password){
reject('Need userId and password to fetch key')
if (!userId || !password) {
reject("Need userId and password to fetch key");
}
db.promise().query('SELECT * FROM user_key WHERE user_id = ? LIMIT 1', [userId])
db.promise()
.query("SELECT * FROM user_key WHERE user_id = ? LIMIT 1", [userId])
.then((rows, fields) => {
const row = rows[0][0];
const row = rows[0][0]
if(!rows[0] || rows[0].length == 0 || rows[0][0] == undefined){
return reject('Row or salt or something not set')
if (!rows[0] || rows[0].length == 0 || rows[0][0] == undefined) {
return reject("Row or salt or something not set");
}
const masterKey = cs.decrypt(password, row['salt'], row['key'])
const masterKey = cs.decrypt(password, row["salt"], row["key"]);
if(masterKey == null){
return reject('Unable to decrypt key')
if (masterKey == null) {
return reject("Unable to decrypt key");
}
return resolve(masterKey)
})
})
}
return resolve(masterKey);
});
});
};
User.generateKeypair = (userId, masterKey) => {
let publicKey = null
let privateKey = null
let publicKey = null;
let privateKey = null;
return new Promise((resolve, reject) => {
db.promise().query('SELECT * FROM user_key WHERE user_id = ?', [userId])
db.promise()
.query("SELECT * FROM user_key WHERE user_id = ?", [userId])
.then((rows, fields) => {
const row = rows[0][0];
const row = rows[0][0]
const salt = row["salt"];
publicKey = row["public_key"];
privateKey = row["private_key_encrypted"];
const salt = row['salt']
publicKey = row['public_key']
privateKey = row['private_key_encrypted']
if(row['public_key'] == null){
const keyPair = crypto.generateKeyPairSync('rsa', {
if (row["public_key"] == null) {
const keyPair = crypto.generateKeyPairSync("rsa", {
modulusLength: 1024,
publicKeyEncoding: {
type: 'spki',
format: 'pem'
type: "spki",
format: "pem",
},
privateKeyEncoding: {
type: 'pkcs8',
format: 'pem'
}
})
type: "pkcs8",
format: "pem",
},
});
publicKey = keyPair.publicKey
privateKey = keyPair.privateKey
const privateKeyEncrypted = cs.encrypt(masterKey, salt, privateKey)
publicKey = keyPair.publicKey;
privateKey = keyPair.privateKey;
const privateKeyEncrypted = cs.encrypt(masterKey, salt, privateKey);
db.promise()
.query(
'UPDATE user_key SET `public_key` = ?, `private_key_encrypted` = ? WHERE user_id = ?;',
[publicKey, privateKeyEncrypted, userId]
"UPDATE user_key SET `public_key` = ?, `private_key_encrypted` = ? WHERE user_id = ?;",
[publicKey, privateKeyEncrypted, userId],
)
.then((rows, fields)=>{
return resolve({publicKey, privateKey})
})
.then((rows, fields) => {
return resolve({ publicKey, privateKey });
});
} else {
//Decrypt private key
privateKey = cs.decrypt(masterKey, salt, privateKey)
privateKey = cs.decrypt(masterKey, salt, privateKey);
return resolve({publicKey, privateKey})
return resolve({ publicKey, privateKey });
}
})
})
}
});
});
};
User.getPublicKey = (userId) => {
return new Promise((resolve, reject) => {
db.promise().query('SELECT public_key FROM user_key WHERE user_id = ?', [userId])
db.promise()
.query("SELECT public_key FROM user_key WHERE user_id = ?", [userId])
.then((rows, fields) => {
const row = rows[0][0]
return resolve(row['public_key'])
})
})
}
const row = rows[0][0];
return resolve(row["public_key"]);
});
});
};
User.getPrivateKey = (userId, masterKey) => {
return new Promise((resolve, reject) => {
db.promise().query('SELECT salt, private_key_encrypted FROM user_key WHERE user_id = ?', [userId])
db.promise()
.query(
"SELECT salt, private_key_encrypted FROM user_key WHERE user_id = ?",
[userId],
)
.then((rows, fields) => {
const row = rows[0][0];
const row = rows[0][0]
const salt = row['salt']
privateKey = row['private_key_encrypted']
const salt = row["salt"];
privateKey = row["private_key_encrypted"];
//Decrypt private key
privateKey = cs.decrypt(masterKey, salt, privateKey)
privateKey = cs.decrypt(masterKey, salt, privateKey);
return resolve(privateKey)
})
})
}
return resolve(privateKey);
});
});
};
User.getByUserName = (username) => {
return new Promise((resolve, reject) => {
db.promise().query('SELECT * FROM user WHERE username = ? LIMIT 1', [username.toLowerCase()])
db.promise()
.query("SELECT * FROM user WHERE username = ? LIMIT 1", [
username.toLowerCase(),
])
.then((rows, fields) => {
resolve(rows[0][0])
})
})
}
resolve(rows[0][0]);
});
});
};
User.changePassword = (userId, oldPass, newPass) => {
return new Promise((resolve, reject) => {
User.getMasterKey(userId, oldPass)
.then(masterKey => {
User.getPrivateKey(userId, masterKey)
.then(privateKey => {
.then((masterKey) => {
User.getPrivateKey(userId, masterKey).then((privateKey) => {
//If success, user has correct password
// Generate new master pass, encrypt with new password
// const masterPassword = cs.createSmallSalt()
const salt = cs.createSmallSalt()
const encryptedMasterPassword = cs.encrypt(newPass, salt, masterKey)
const encryptedPrivateKey = cs.encrypt(masterKey, salt, privateKey)
const salt = cs.createSmallSalt();
const encryptedMasterPassword = cs.encrypt(newPass, salt, masterKey);
const encryptedPrivateKey = cs.encrypt(masterKey, salt, privateKey);
db.promise()
.query(
'UPDATE user_key SET salt = ?, `key` = ?, private_key_encrypted = ? WHERE user_id = ? LIMIT 1',
[salt, encryptedMasterPassword, encryptedPrivateKey, userId]
).then((r,f) => {
"UPDATE user_key SET salt = ?, `key` = ?, private_key_encrypted = ? WHERE user_id = ? LIMIT 1",
[salt, encryptedMasterPassword, encryptedPrivateKey, userId],
)
.then((r, f) => {
//Create login using password
let shasum = crypto.createHash('sha512') //Prepare Hash
const saltString = shasum.digest('hex')
const passwordSalt = Buffer.from(saltString, 'binary') //Generate Salt hash
const iterations = 25000
let shasum = crypto.createHash("sha512"); //Prepare Hash
const saltString = shasum.digest("hex");
const passwordSalt = Buffer.from(saltString, "binary"); //Generate Salt hash
const iterations = 25000;
crypto.pbkdf2(newPass, passwordSalt, iterations, 512, 'sha512', function(err, delivered_key) {
crypto.pbkdf2(
newPass,
passwordSalt,
iterations,
512,
"sha512",
function (err, delivered_key) {
const deliveredPass = delivered_key.toString("hex");
const deliveredPass = delivered_key.toString('hex')
db.promise().query('UPDATE user SET password = ?, salt = ? WHERE id = ? LIMIT 1', [deliveredPass, passwordSalt, userId])
.then((r,f) => {
return resolve(true)
db.promise()
.query(
"UPDATE user SET password = ?, salt = ? WHERE id = ? LIMIT 1",
[deliveredPass, passwordSalt, userId],
)
.then((r, f) => {
return resolve(true);
});
},
);
});
});
})
})
})
})
})
.catch(error => {
resolve(false)
})
})
}
.catch((error) => {
resolve(false);
});
});
};
User.revokeActiveSessions = (userId, sessionId) => {
return new Promise((resolve, reject) => {
const userHash = cs.hash(String(userId)).toString("base64");
const userHash = cs.hash(String(userId)).toString('base64')
db.promise().query('DELETE FROM user_active_session WHERE user_hash = ? AND session_id != ?', [userHash, sessionId])
.then((r,f) => {
resolve(true)
})
})
}
db.promise()
.query(
"DELETE FROM user_active_session WHERE user_hash = ? AND session_id != ?",
[userHash, sessionId],
)
.then((r, f) => {
resolve(true);
});
});
};
User.deleteUser = (userId, password) => {
if(!userId || !password){
if (!userId || !password) {
return new Promise((resolve, reject) => {
return resolve('Missing User ID or Password. No Action Taken.')
})
return resolve("Missing User ID or Password. No Action Taken.");
});
}
//Verify user is correct by decryptig master key with password
let deletePromises = []
let deletePromises = [];
//Delete all notes and raw text
let noteDelete = db.promise().query(`
let noteDelete = db.promise().query(
`
DELETE note, note_raw_text
FROM note
JOIN note_raw_text ON (note_raw_text.id = note.note_raw_text_id)
WHERE note.user_id = ?
`,[userId])
deletePromises.push(noteDelete)
`,
[userId],
);
deletePromises.push(noteDelete);
//Delete user entry
let userDelete = db.promise().query(`
let userDelete = db.promise().query(
`
DELETE FROM user WHERE id = ?
`,[userId])
deletePromises.push(userDelete)
`,
[userId],
);
deletePromises.push(userDelete);
//Delete user_key, encrypted search index
let tables = ['user_key', 'user_encrypted_search_index']
tables.forEach(tableName => {
const query = `DELETE FROM ${tableName} WHERE user_id = ?`
const deleteQuery = db.promise().query(query, [userId])
deletePromises.push(deleteQuery)
})
let tables = ["user_key", "user_encrypted_search_index"];
tables.forEach((tableName) => {
const query = `DELETE FROM ${tableName} WHERE user_id = ?`;
const deleteQuery = db.promise().query(query, [userId]);
deletePromises.push(deleteQuery);
});
//Remove all note attachments and files
return Promise.all(deletePromises)
}
return Promise.all(deletePromises);
};
server/tests/fixtures/html/README.md
Vendored
+7
View File
@@ -0,0 +1,7 @@
# HTML fixtures
Frozen, hand-trimmed snapshots used by `server/tests/helpers/SiteScrape.test.js`.
These are **not** fetched live. Each file isolates one parsing situation so a
failure points at a specific branch. Add a new file rather than growing an
existing one when covering a new case.
server/tests/fixtures/html/apple-touch-icon.html
Vendored
+9
View File
@@ -0,0 +1,9 @@
<!doctype html>
<html>
<head>
<title>Icons By Size</title>
<link rel="icon" href="https://example.com/small.png" sizes="16x16">
<link rel="apple-touch-icon" href="https://example.com/touch-180.png" sizes="180x180">
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/extensionless-og-image.html
Vendored
+8
View File
@@ -0,0 +1,8 @@
<!doctype html>
<html>
<head>
<title>Generated OG Image</title>
<meta property="og:image" content="https://example.com/api/og?title=Hello&amp;theme=dark">
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/h1-fallback.html
Vendored
+5
View File
@@ -0,0 +1,5 @@
<!doctype html>
<html>
<head></head>
<body><h1>Heading Is All We Have</h1><p>No title element, no meta tags.</p></body>
</html>
server/tests/fixtures/html/imgs-missing-src.html
Vendored
+10
View File
@@ -0,0 +1,10 @@
<!doctype html>
<html>
<head><title>Broken Images</title></head>
<body>
<img>
<img alt="no src here">
<img src="">
<img data-src="https://cdn.example.com/lazy.jpg">
</body>
</html>
server/tests/fixtures/html/jsonld-article.html
Vendored
+15
View File
@@ -0,0 +1,15 @@
<!doctype html>
<html>
<head>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "NewsArticle",
"headline": "JSON-LD Headline",
"description": "Described in structured data.",
"image": ["https://cdn.example.com/ld-primary.jpg", "https://cdn.example.com/ld-second.jpg"]
}
</script>
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/jsonld-graph.html
Vendored
+15
View File
@@ -0,0 +1,15 @@
<!doctype html>
<html>
<head>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@graph": [
{ "@type": "Organization", "name": "Not The Article" },
{ "@type": "BlogPosting", "headline": "Graph Headline", "image": { "url": "https://cdn.example.com/graph.png" } }
]
}
</script>
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/jsonld-malformed.html
Vendored
+11
View File
@@ -0,0 +1,11 @@
<!doctype html>
<html>
<head>
<title>Survived Bad JSON</title>
<script type="application/ld+json">
{ "@type": "Article", "headline": "unterminated,,, }
</script>
<script type="application/ld+json"></script>
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/keyword-content.html
Vendored
+16
View File
@@ -0,0 +1,16 @@
<!doctype html>
<html>
<head><title>Beekeeping Guide</title></head>
<body>
<nav>Home About Contact Login Signup Navigation Navigation</nav>
<script>var tracking = "tracking tracking tracking tracking tracking";</script>
<style>.beehive { color: red }</style>
<article class="content">
<p>Beekeeping is rewarding. Beekeeping requires patience. A beehive needs space.</p>
<p>The beehive must stay dry. Honey harvest comes later. Honey harvest rewards patience.</p>
<p>Beekeeping beginners should buy a beehive and study honey production carefully.</p>
<p>Honey production depends on the beehive and on steady beekeeping habits.</p>
</article>
<footer>Footer footer footer links copyright</footer>
</body>
</html>
server/tests/fixtures/html/malformed-unclosed.html
Vendored
+7
View File
@@ -0,0 +1,7 @@
<!doctype html>
<html>
<head><title>Unclosed Tags
<body>
<div><p>Paragraph never closed
<ul><li>item one<li>item two
</html>
server/tests/fixtures/html/no-metadata.html
Vendored
+2
View File
@@ -0,0 +1,2 @@
<!doctype html>
<html><head></head><body></body></html>
server/tests/fixtures/html/og-complete.html
Vendored
+14
View File
@@ -0,0 +1,14 @@
<!doctype html>
<html lang="en">
<head>
<title>Fallback Title Nobody Should See</title>
<meta property="og:title" content="The Real Title">
<meta property="og:description" content="A proper social description of the page.">
<meta property="og:site_name" content="Example Journal">
<meta property="og:image" content="https://cdn.example.com/hero.jpg">
<meta property="og:url" content="https://example.com/canonical-post">
<meta name="description" content="A lesser meta description.">
<link rel="canonical" href="https://example.com/canonical-post">
</head>
<body><article><p>Body text.</p></article></body>
</html>
server/tests/fixtures/html/protocol-relative-image.html
Vendored
+8
View File
@@ -0,0 +1,8 @@
<!doctype html>
<html>
<head>
<title>Protocol Relative</title>
<meta property="og:image" content="//cdn.example.com/proto.jpg">
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/relative-images.html
Vendored
+9
View File
@@ -0,0 +1,9 @@
<!doctype html>
<html>
<head>
<title>Relative Assets</title>
<link rel="canonical" href="https://example.com/blog/2026/post">
<meta property="og:image" content="../images/hero.png">
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/root-relative-icon.html
Vendored
+8
View File
@@ -0,0 +1,8 @@
<!doctype html>
<html>
<head>
<title>Icon Only</title>
<link rel="shortcut icon" href="/favicon.ico">
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/sized-images.html
Vendored
+9
View File
@@ -0,0 +1,9 @@
<!doctype html>
<html>
<head><title>Sized</title></head>
<body>
<img src="https://cdn.example.com/thumb.jpg" width="80" height="60">
<img src="https://cdn.example.com/big.jpg" width="1200" height="800">
<img src="https://cdn.example.com/mid.jpg" width="400" height="300">
</body>
</html>
server/tests/fixtures/html/srcset.html
Vendored
+7
View File
@@ -0,0 +1,7 @@
<!doctype html>
<html>
<head><title>Srcset</title></head>
<body>
<img srcset="https://cdn.example.com/small.jpg 320w, https://cdn.example.com/large.jpg 1600w, https://cdn.example.com/medium.jpg 800w">
</body>
</html>
server/tests/fixtures/html/svg-only-image.html
Vendored
+8
View File
@@ -0,0 +1,8 @@
<!doctype html>
<html>
<head>
<title>SVG Only</title>
<meta property="og:image" content="https://cdn.example.com/logo.svg">
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/title-only.html
Vendored
+5
View File
@@ -0,0 +1,5 @@
<!doctype html>
<html>
<head><title> Just A Title </title></head>
<body><h1>An H1 Heading</h1><p>No metadata here at all.</p></body>
</html>
server/tests/fixtures/html/tracking-pixels.html
Vendored
+9
View File
@@ -0,0 +1,9 @@
<!doctype html>
<html>
<head><title>Pixels</title></head>
<body>
<img src="https://analytics.example.net/1x1.gif" width="1" height="1">
<img src="https://cdn.example.com/spacer.gif">
<img src="https://cdn.example.com/real-photo.jpg" width="800" height="600">
</body>
</html>
server/tests/fixtures/html/twitter-only.html
Vendored
+10
View File
@@ -0,0 +1,10 @@
<!doctype html>
<html>
<head>
<title>Document Title</title>
<meta name="twitter:title" content="Twitter Card Title">
<meta name="twitter:description" content="Twitter card description.">
<meta name="twitter:image" content="https://cdn.example.com/twitter.png">
</head>
<body><p>x</p></body>
</html>
server/tests/fixtures/html/webp-og-image.html
Vendored
+8
View File
@@ -0,0 +1,8 @@
<!doctype html>
<html>
<head>
<title>Modern Formats</title>
<meta property="og:image" content="https://cdn.example.com/hero.webp">
</head>
<body><p>x</p></body>
</html>
server/tests/helpers/HostQueue.test.js
+318
View File
@@ -0,0 +1,318 @@
require('module-alias/register')
const HostQueue = require('@helpers/HostQueue')
//
// A controllable clock that also drives the queue's own timers, so advancing
// time is instant and spacing assertions are exact rather than flaky.
//
const makeClock = () => {
let t = 1000
let nextId = 1
const timers = new Map()
return {
now: () => t,
setTimeoutFn: (fn, ms) => {
const id = nextId++
timers.set(id, { fn, dueAt: t + ms })
return id
},
clearTimeoutFn: (id) => { timers.delete(id) },
advance: (ms) => {
t += ms
for (const [id, timer] of [...timers.entries()]) {
if (timer.dueAt <= t) {
timers.delete(id)
timer.fn()
}
}
},
}
}
const wireClock = (clock) => ({
now: clock.now,
setTimeoutFn: clock.setTimeoutFn,
clearTimeoutFn: clock.clearTimeoutFn,
})
const deferred = () => {
let resolve = null
let reject = null
const promise = new Promise((res, rej) => { resolve = res; reject = rej })
return { promise, resolve, reject }
}
describe('global concurrency', () => {
test('never exceeds the cap', async () => {
const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
let active = 0
let peak = 0
const gates = []
const jobs = Array.from({ length: 6 }, (unused, i) => {
const gate = deferred()
gates.push(gate)
return queue.schedule(`host${i}.com`, async () => {
active++
peak = Math.max(peak, active)
await gate.promise
active--
return i
})
})
// Let the first batch start.
await new Promise(resolve => setImmediate(resolve))
expect(peak).toBe(2)
gates.forEach(gate => gate.resolve())
await Promise.all(jobs)
expect(peak).toBe(2)
})
test('runs different hosts in parallel up to the cap', async () => {
const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
let active = 0
let peak = 0
const gates = []
const jobs = ['a.com', 'b.com', 'c.com'].map(host => {
const gate = deferred()
gates.push(gate)
return queue.schedule(host, async () => {
active++
peak = Math.max(peak, active)
await gate.promise
active--
})
})
await new Promise(resolve => setImmediate(resolve))
expect(peak).toBe(3)
gates.forEach(gate => gate.resolve())
await Promise.all(jobs)
})
})
describe('per-host serialisation', () => {
test('runs only one request per host at a time', async () => {
const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
let active = 0
let peak = 0
const jobs = [1, 2, 3].map(() => queue.schedule('same.com', async () => {
active++
peak = Math.max(peak, active)
await new Promise(resolve => setImmediate(resolve))
active--
}))
await Promise.all(jobs)
expect(peak).toBe(1)
})
test('preserves FIFO order within a host', async () => {
const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
const order = []
await Promise.all([1, 2, 3, 4].map(n =>
queue.schedule('same.com', async () => { order.push(n) })
))
expect(order).toEqual([1, 2, 3, 4])
})
})
describe('politeness spacing', () => {
test('leaves a gap between two requests to the same host', async () => {
const clock = makeClock()
const queue = new HostQueue.Queue({
globalConcurrency: 4,
minSpacingMs: 1000,
maxSpacingMs: 1000,
...wireClock(clock),
})
const startedAt = []
const first = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
await first
const second = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
// The second job must not have run yet - the host is spaced out.
await new Promise(resolve => setImmediate(resolve))
expect(startedAt).toHaveLength(1)
clock.advance(1000)
await second
expect(startedAt).toHaveLength(2)
expect(startedAt[1] - startedAt[0]).toBeGreaterThanOrEqual(1000)
})
test('spacing on one host does not delay a different host', async () => {
const clock = makeClock()
const queue = new HostQueue.Queue({
globalConcurrency: 4,
minSpacingMs: 5000,
maxSpacingMs: 5000,
...wireClock(clock),
})
await queue.schedule('slow.com', async () => {})
// slow.com is now parked for 5s. other.com must be unaffected.
const ran = []
await queue.schedule('other.com', async () => { ran.push('other') })
expect(ran).toEqual(['other'])
})
test('honours an explicit spacing override (robots Crawl-delay)', async () => {
const clock = makeClock()
const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0, ...wireClock(clock) })
await queue.schedule('slow.com', async () => {}, { spacingMs: 3000 })
const ran = []
const second = queue.schedule('slow.com', async () => { ran.push(1) })
await new Promise(resolve => setImmediate(resolve))
expect(ran).toEqual([])
clock.advance(3000)
await second
expect(ran).toEqual([1])
})
})
describe('error handling', () => {
test('a rejecting task does not wedge the queue', async () => {
const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
await expect(queue.schedule('a.com', async () => { throw new Error('boom') }))
.rejects.toThrow('boom')
await expect(queue.schedule('a.com', async () => 'fine')).resolves.toBe('fine')
})
test('a task that throws synchronously is caught', async () => {
const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
await expect(queue.schedule('a.com', () => { throw new Error('sync boom') }))
.rejects.toThrow('sync boom')
expect(queue.inflight()).toBe(0)
})
test('one failing host does not block others', async () => {
const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
const results = await Promise.allSettled([
queue.schedule('bad.com', async () => { throw new Error('nope') }),
queue.schedule('good.com', async () => 'ok'),
])
expect(results[0].status).toBe('rejected')
expect(results[1].value).toBe('ok')
})
})
describe('drain and stop', () => {
test('drain resolves when everything has finished', async () => {
const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
const done = []
;[1, 2, 3].forEach(n => queue.schedule(`h${n}.com`, async () => { done.push(n) }).catch(() => {}))
await queue.drain()
expect(done).toHaveLength(3)
expect(queue.size()).toBe(0)
expect(queue.inflight()).toBe(0)
})
test('drain resolves immediately on an idle queue', async () => {
const queue = new HostQueue.Queue()
await expect(queue.drain()).resolves.toBeUndefined()
})
test('stop rejects queued work and refuses new work', async () => {
const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
const gate = deferred()
const first = queue.schedule('a.com', () => gate.promise)
const queued = queue.schedule('b.com', async () => 'never')
await new Promise(resolve => setImmediate(resolve))
queue.stop()
await expect(queued).rejects.toThrow('stopped')
await expect(queue.schedule('c.com', async () => 'x')).rejects.toThrow('stopped')
gate.resolve('done')
await expect(first).resolves.toBe('done')
})
})
describe('event loop', () => {
//
// Regression: the politeness timer used to be unref'd, so if nothing else
// held the event loop open Node exited and queued jobs were abandoned with
// their promises never settling.
//
test('a spaced job still resolves with nothing else holding the loop open', async () => {
const queue = new HostQueue.Queue({ minSpacingMs: 25, maxSpacingMs: 25 })
await queue.schedule('same.com', async () => 'first')
// Only the internal timer can resume this one.
await expect(queue.schedule('same.com', async () => 'second')).resolves.toBe('second')
})
test('stop() clears the pending timer so shutdown is not blocked', async () => {
const queue = new HostQueue.Queue({ minSpacingMs: 10000, maxSpacingMs: 10000 })
await queue.schedule('same.com', async () => 'first')
const queued = queue.schedule('same.com', async () => 'never')
queue.stop()
await expect(queued).rejects.toThrow('stopped')
})
})
describe('stats', () => {
test('reports pending, running and host counts', async () => {
const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
const gate = deferred()
const first = queue.schedule('a.com', () => gate.promise)
const second = queue.schedule('b.com', async () => {})
await new Promise(resolve => setImmediate(resolve))
const stats = queue.stats()
expect(stats.running).toBe(1)
expect(stats.pending).toBe(1)
gate.resolve()
await Promise.all([first, second])
})
})
server/tests/helpers/HttpFetch.test.js
+479
View File
@@ -0,0 +1,479 @@
require('module-alias/register')
const HttpFetch = require('@helpers/HttpFetch')
const BrowserProfiles = require('@helpers/BrowserProfiles')
const FetchMock = require('../support/fetchMock')
const dns = require('dns').promises
let dnsSpy = null
beforeEach(() => {
FetchMock.install()
// Every hostname in these tests resolves to one public address, so the
// safety gate passes and we are testing HttpFetch rather than DNS.
dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
// No real wall-time in backoff.
HttpFetch.__setSleep(() => Promise.resolve())
})
afterEach(() => {
FetchMock.restore()
dnsSpy.mockRestore()
HttpFetch.__resetSleep()
})
describe('fetchDocument happy path', () => {
test('returns decoded html, status and final url', async () => {
FetchMock.route('https://example.com/', FetchMock.html('<title>Hello</title>'))
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.status).toBe(200)
expect(result.html).toContain('<title>Hello</title>')
expect(result.finalUrl).toBe('https://example.com/')
expect(result.notModified).toBe(false)
})
test('surfaces etag and last-modified for later conditional requests', async () => {
FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>', {
headers: { etag: 'W/"abc"', 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT' },
}))
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.etag).toBe('W/"abc"')
expect(result.lastModified).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
})
})
describe('fetchDocument browser identity', () => {
test('sends a complete, coherent header set and never claims to be Googlebot', async () => {
FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>'))
await HttpFetch.fetchDocument('https://example.com/')
const headers = FetchMock.calls()[0].headers
expect(headers['User-Agent']).toMatch(/^Mozilla\/5\.0/)
expect(headers['User-Agent']).not.toMatch(/Googlebot/i)
expect(headers['Accept']).toContain('text/html')
expect(headers['Accept-Language']).toBeTruthy()
expect(headers['Accept-Encoding']).toContain('gzip')
})
test('gives the same host the same profile across calls', async () => {
FetchMock.route(/example\.com/, () => FetchMock.html('<p>x</p>'))
const first = await HttpFetch.fetchDocument('https://example.com/a')
const second = await HttpFetch.fetchDocument('https://example.com/b')
expect(first.profileId).toBe(second.profileId)
})
test('pairs sec-ch-ua only with Chromium profiles', () => {
const chrome = BrowserProfiles.profileById('chrome-win')
const firefox = BrowserProfiles.profileById('firefox-win')
const safari = BrowserProfiles.profileById('safari-mac')
const chromeHeaders = BrowserProfiles.documentHeaders(chrome, { url: 'https://a.com/' })
const firefoxHeaders = BrowserProfiles.documentHeaders(firefox, { url: 'https://a.com/' })
const safariHeaders = BrowserProfiles.documentHeaders(safari, { url: 'https://a.com/' })
expect(chromeHeaders['sec-ch-ua']).toBeTruthy()
expect(chromeHeaders['Sec-Fetch-Mode']).toBe('navigate')
// Firefox sends Sec-Fetch-* but no client hints.
expect(firefoxHeaders['sec-ch-ua']).toBeUndefined()
expect(firefoxHeaders['Sec-Fetch-Mode']).toBe('navigate')
// Safari sends neither.
expect(safariHeaders['sec-ch-ua']).toBeUndefined()
expect(safariHeaders['Sec-Fetch-Mode']).toBeUndefined()
})
test('never pairs a chrome UA with a firefox accept header', () => {
BrowserProfiles.PROFILES.forEach(profile => {
if (profile.family === 'chrome') {
expect(profile.ua).toMatch(/Chrome\//)
expect(profile.accept).toContain('application/signed-exchange')
}
if (profile.family === 'firefox') {
expect(profile.ua).toMatch(/Firefox\//)
expect(profile.accept).not.toContain('signed-exchange')
}
})
})
})
describe('fetchDocument redirects', () => {
test('follows a single hop and reports the final url', async () => {
FetchMock.route('https://example.com/a', FetchMock.redirect('https://example.com/b', 301))
FetchMock.route('https://example.com/b', () => FetchMock.html('<title>B</title>'))
const result = await HttpFetch.fetchDocument('https://example.com/a')
expect(result.finalUrl).toBe('https://example.com/b')
expect(result.html).toContain('B')
})
test('resolves a relative Location header', async () => {
FetchMock.route('https://example.com/a', FetchMock.redirect('/c', 302))
FetchMock.route('https://example.com/c', () => FetchMock.html('<title>C</title>'))
const result = await HttpFetch.fetchDocument('https://example.com/a')
expect(result.finalUrl).toBe('https://example.com/c')
})
test('detects a redirect loop', async () => {
FetchMock.route('https://example.com/a', () => FetchMock.redirect('https://example.com/b', 302))
FetchMock.route('https://example.com/b', () => FetchMock.redirect('https://example.com/a', 302))
await expect(HttpFetch.fetchDocument('https://example.com/a'))
.rejects.toMatchObject({ kind: 'redirect_loop' })
})
test('gives up past the redirect cap', async () => {
let n = 0
FetchMock.route(/https:\/\/example\.com\/hop/, () => FetchMock.redirect(`https://example.com/hop${++n}`, 302))
await expect(HttpFetch.fetchDocument('https://example.com/hop'))
.rejects.toMatchObject({ kind: 'too_many_redirects' })
})
// The SSRF case that matters: the first URL is perfectly public.
test('blocks a redirect pointing at loopback', async () => {
FetchMock.route('https://example.com/a', FetchMock.redirect('http://127.0.0.1:3000/admin', 302))
await expect(HttpFetch.fetchDocument('https://example.com/a'))
.rejects.toMatchObject({ kind: 'blocked' })
})
test('blocks a redirect pointing at the cloud metadata endpoint', async () => {
FetchMock.route('https://example.com/a', FetchMock.redirect('http://169.254.169.254/latest/meta-data/', 302))
await expect(HttpFetch.fetchDocument('https://example.com/a'))
.rejects.toMatchObject({ kind: 'blocked' })
})
test('refuses an https to http downgrade', async () => {
FetchMock.route('https://example.com/a', FetchMock.redirect('http://example.com/a', 302))
await expect(HttpFetch.fetchDocument('https://example.com/a'))
.rejects.toMatchObject({ kind: 'blocked' })
})
})
describe('fetchDocument retries', () => {
test('retries a 500 and succeeds', async () => {
FetchMock.routeSequence('https://example.com/', [
FetchMock.status(500),
FetchMock.html('<title>Recovered</title>'),
])
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.html).toContain('Recovered')
expect(FetchMock.callCount()).toBe(2)
})
test('gives up after exhausting retries', async () => {
FetchMock.route('https://example.com/', () => FetchMock.status(503))
await expect(HttpFetch.fetchDocument('https://example.com/'))
.rejects.toMatchObject({ kind: 'http', status: 503 })
// 1 initial attempt + config.retries
expect(FetchMock.callCount()).toBe(3)
})
test('does NOT retry a 404 - it is an answer, not a failure', async () => {
FetchMock.route('https://example.com/', () => FetchMock.status(404))
await expect(HttpFetch.fetchDocument('https://example.com/'))
.rejects.toMatchObject({ kind: 'http', status: 404 })
expect(FetchMock.callCount()).toBe(1)
})
test('does NOT retry a 403', async () => {
FetchMock.route('https://example.com/', () => FetchMock.status(403))
await expect(HttpFetch.fetchDocument('https://example.com/')).rejects.toMatchObject({ status: 403 })
expect(FetchMock.callCount()).toBe(1)
})
test('honours a short Retry-After on a 429', async () => {
const waits = []
HttpFetch.__setSleep((ms) => { waits.push(ms); return Promise.resolve() })
FetchMock.routeSequence('https://example.com/', [
FetchMock.status(429, { headers: { 'retry-after': '2' } }),
FetchMock.html('<title>OK</title>'),
])
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.html).toContain('OK')
expect(waits[0]).toBe(2000)
})
test('refuses to sit on an hour-long Retry-After', async () => {
FetchMock.route('https://example.com/', () => FetchMock.status(429, { headers: { 'retry-after': '3600' } }))
await expect(HttpFetch.fetchDocument('https://example.com/'))
.rejects.toMatchObject({ kind: 'http', status: 429 })
// Surfaced for the cache to park the URL rather than retried.
expect(FetchMock.callCount()).toBe(1)
})
test('retries a network error', async () => {
FetchMock.routeSequence('https://example.com/', [
FetchMock.networkError('ECONNRESET'),
FetchMock.html('<title>Up</title>'),
])
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.html).toContain('Up')
})
})
describe('fetchDocument limits and gates', () => {
test('times out on a server that never responds', async () => {
FetchMock.route('https://example.com/', () => FetchMock.hang())
await expect(HttpFetch.fetchDocument('https://example.com/', { timeoutMs: 60, retries: 0 }))
.rejects.toMatchObject({ kind: 'timeout' })
})
test('aborts a body that exceeds the cap mid-stream', async () => {
FetchMock.route('https://example.com/', () => FetchMock.oversized())
await expect(HttpFetch.fetchDocument('https://example.com/', { maxBytes: 128 * 1024, retries: 0 }))
.rejects.toMatchObject({ kind: 'too_large' })
})
test('rejects an oversized declared content-length without reading the body', async () => {
FetchMock.route('https://example.com/', () => FetchMock.html('<p>small</p>', {
headers: { 'content-length': String(50 * 1024 * 1024) },
}))
await expect(HttpFetch.fetchDocument('https://example.com/', { retries: 0 }))
.rejects.toMatchObject({ kind: 'too_large' })
})
test('refuses a non-html content type before parsing', async () => {
FetchMock.route('https://example.com/f.pdf', () => FetchMock.buffer(Buffer.alloc(16), {
contentType: 'application/pdf',
}))
await expect(HttpFetch.fetchDocument('https://example.com/f.pdf', { retries: 0 }))
.rejects.toMatchObject({ kind: 'bad_content_type' })
})
test('refuses a binary masquerading behind an image content type', async () => {
FetchMock.route('https://example.com/x', () => FetchMock.buffer(Buffer.alloc(16), {
contentType: 'image/png',
}))
await expect(HttpFetch.fetchDocument('https://example.com/x', { retries: 0 }))
.rejects.toMatchObject({ kind: 'bad_content_type' })
})
})
describe('fetchDocument charset handling', () => {
test('decodes windows-1252 declared in the content-type header', async () => {
// 0x93/0x94 are curly quotes in cp1252 and invalid in utf-8.
const body = Buffer.concat([
Buffer.from('<title>', 'latin1'),
Buffer.from([0x93]),
Buffer.from('caf', 'latin1'),
Buffer.from([0xe9, 0x94]),
Buffer.from('</title>', 'latin1'),
])
FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
contentType: 'text/html; charset=windows-1252',
}))
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.html).toContain('“café”')
expect(result.charset).toBe('windows-1252')
})
test('falls back to a meta charset when the header says nothing', async () => {
const body = Buffer.concat([
Buffer.from('<meta charset="windows-1252"><title>caf', 'latin1'),
Buffer.from([0xe9]),
Buffer.from('</title>', 'latin1'),
])
FetchMock.route('https://example.com/', () => FetchMock.buffer(body, { contentType: 'text/html' }))
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.html).toContain('café')
})
test('header charset wins over the meta tag', async () => {
const body = Buffer.from('<meta charset="windows-1252"><title>café</title>', 'utf8')
FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
contentType: 'text/html; charset=utf-8',
}))
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.html).toContain('café')
})
test('survives a bogus charset label', async () => {
FetchMock.route('https://example.com/', () => FetchMock.buffer(Buffer.from('<p>ok</p>'), {
contentType: 'text/html; charset=not-a-real-charset',
}))
const result = await HttpFetch.fetchDocument('https://example.com/')
expect(result.html).toContain('ok')
})
})
describe('fetchDocument conditional requests', () => {
test('sends If-None-Match and If-Modified-Since when given', async () => {
FetchMock.route('https://example.com/', () => FetchMock.html('<p>x</p>'))
await HttpFetch.fetchDocument('https://example.com/', {
conditional: { etag: 'W/"v1"', lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT' },
})
const headers = FetchMock.calls()[0].headers
expect(headers['If-None-Match']).toBe('W/"v1"')
expect(headers['If-Modified-Since']).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
})
test('reports a 304 as notModified with no body', async () => {
FetchMock.route('https://example.com/', () => FetchMock.status(304))
const result = await HttpFetch.fetchDocument('https://example.com/', {
conditional: { etag: 'W/"v1"' },
})
expect(result.notModified).toBe(true)
expect(result.status).toBe(304)
expect(result.html).toBeNull()
})
})
describe('fetchBinary', () => {
const pngMagic = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
test('returns the buffer for a real image', async () => {
FetchMock.route('https://example.com/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))
const result = await HttpFetch.fetchBinary('https://example.com/i.png')
expect(result.mime).toBe('image/png')
expect(result.buffer.equals(pngMagic)).toBe(true)
})
// The old code wrote a 404 HTML error page to disk and handed it to gm.
test('refuses an html error page served where an image was expected', async () => {
FetchMock.route('https://example.com/i.png', () => FetchMock.html('<h1>404</h1>'))
await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
.rejects.toMatchObject({ kind: 'bad_content_type' })
})
test('propagates a 404 status rather than writing anything', async () => {
FetchMock.route('https://example.com/i.png', () => FetchMock.status(404))
await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
.rejects.toMatchObject({ kind: 'http', status: 404 })
})
test('enforces the image size cap', async () => {
FetchMock.route('https://example.com/i.png', () => FetchMock.oversized('image/png'))
await expect(HttpFetch.fetchBinary('https://example.com/i.png', { maxBytes: 64 * 1024, retries: 0 }))
.rejects.toMatchObject({ kind: 'too_large' })
})
test('rejects svg by default', async () => {
FetchMock.route('https://example.com/i.svg', () => FetchMock.buffer(Buffer.from('<svg/>'), {
contentType: 'image/svg+xml',
}))
await expect(HttpFetch.fetchBinary('https://example.com/i.svg', { retries: 0 }))
.rejects.toMatchObject({ kind: 'bad_content_type' })
})
test('sends image-appropriate fetch metadata and a page referer', async () => {
FetchMock.route('https://cdn.example.net/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))
await HttpFetch.fetchBinary('https://cdn.example.net/i.png', { pageUrl: 'https://example.com/post' })
const headers = FetchMock.calls()[0].headers
expect(headers['Referer']).toBe('https://example.com/post')
expect(headers['Accept']).toContain('image/')
if (headers['Sec-Fetch-Dest']) {
expect(headers['Sec-Fetch-Dest']).toBe('image')
expect(headers['Sec-Fetch-Site']).toBe('cross-site')
}
})
})
describe('transient vs terminal safety failures', () => {
// A private address is a permanent refusal; a DNS failure means the host
// may just be down. Classifying both as "blocked" parked transient
// failures for the full cache TTL.
test('classifies a private address as blocked', async () => {
FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
await expect(HttpFetch.fetchDocument('http://10.0.0.1/', { retries: 0 }))
.rejects.toMatchObject({ kind: 'blocked', reason: 'private_ip' })
})
test('classifies a DNS failure as network, not blocked', async () => {
dnsSpy.mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
await expect(HttpFetch.fetchDocument('https://gone.example/', { retries: 0 }))
.rejects.toMatchObject({ kind: 'network', reason: 'dns_failed' })
})
test('retries a DNS failure, since it may be transient', async () => {
let attempts = 0
dnsSpy.mockImplementation(async () => {
attempts++
if (attempts < 2) { throw Object.assign(new Error('nope'), { code: 'ENOTFOUND' }) }
return [{ address: '93.184.216.34', family: 4 }]
})
FetchMock.route('https://flaky.example/', () => FetchMock.html('<title>Back up</title>'))
const result = await HttpFetch.fetchDocument('https://flaky.example/')
expect(result.html).toContain('Back up')
expect(attempts).toBe(2)
})
test('does NOT retry a private address', async () => {
FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
await expect(HttpFetch.fetchDocument('http://127.0.0.1/')).rejects.toMatchObject({ kind: 'blocked' })
expect(FetchMock.callCount()).toBe(0)
})
})
describe('safety gate on the initial url', () => {
test('refuses to open a connection to a private address at all', async () => {
FetchMock.route(/.*/, () => FetchMock.html('<p>should never be reached</p>'))
await expect(HttpFetch.fetchDocument('http://127.0.0.1:3000/', { retries: 0 }))
.rejects.toMatchObject({ kind: 'blocked' })
expect(FetchMock.callCount()).toBe(0)
})
})
server/tests/helpers/Robots.test.js
+173
View File
@@ -0,0 +1,173 @@
require('module-alias/register')
const Robots = require('@helpers/Robots')
const config = require('@config/scrape')
const FetchMock = require('../support/fetchMock')
const dns = require('dns').promises
let dnsSpy = null
const UA = config.robotsUaToken
beforeEach(() => {
FetchMock.install()
Robots.__resetCache()
dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
})
afterEach(() => {
FetchMock.restore()
dnsSpy.mockRestore()
Robots.__resetCache()
})
const serveRobots = (body, origin = 'https://example.com') => {
FetchMock.route(`${origin}/robots.txt`, () => new Response(body, {
status: 200,
headers: { 'content-type': 'text/plain' },
}))
}
describe('Robots.parse', () => {
test('groups consecutive user-agent lines into one rule block', () => {
const rules = Robots.parse([
'User-agent: alpha',
'User-agent: beta',
'Disallow: /private',
].join('\n'))
expect(rules.groups).toHaveLength(1)
expect(rules.groups[0].agents).toEqual(['alpha', 'beta'])
expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/private' }])
})
test('ignores comments and blank lines', () => {
const rules = Robots.parse('# a comment\n\nUser-agent: *\nDisallow: /x # trailing\n')
expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/x' }])
})
test('treats an empty Disallow as no restriction', () => {
const rules = Robots.parse('User-agent: *\nDisallow:')
expect(rules.groups[0].rules).toHaveLength(0)
})
test('parses crawl-delay', () => {
const rules = Robots.parse('User-agent: *\nCrawl-delay: 2.5')
expect(rules.groups[0].crawlDelayMs).toBe(2500)
})
test('an empty file allows everything', () => {
expect(Robots.parse('').allowAll).toBe(true)
expect(Robots.parse(null).allowAll).toBe(true)
})
})
describe('Robots.isPathAllowed', () => {
test('respects a simple disallow', () => {
const rules = Robots.parse('User-agent: *\nDisallow: /private')
expect(Robots.isPathAllowed(rules, '/private/x', UA)).toBe(false)
expect(Robots.isPathAllowed(rules, '/public/x', UA)).toBe(true)
})
test('longest match wins, so a nested Allow overrides a broad Disallow', () => {
const rules = Robots.parse('User-agent: *\nDisallow: /a\nAllow: /a/b')
expect(Robots.isPathAllowed(rules, '/a/x', UA)).toBe(false)
expect(Robots.isPathAllowed(rules, '/a/b/c', UA)).toBe(true)
})
test('Allow beats Disallow at equal specificity', () => {
const rules = Robots.parse('User-agent: *\nDisallow: /p\nAllow: /p')
expect(Robots.isPathAllowed(rules, '/p', UA)).toBe(true)
})
test('supports the * wildcard', () => {
const rules = Robots.parse('User-agent: *\nDisallow: /*.pdf')
expect(Robots.isPathAllowed(rules, '/docs/report.pdf', UA)).toBe(false)
expect(Robots.isPathAllowed(rules, '/docs/report.html', UA)).toBe(true)
})
test('supports the $ anchor', () => {
const rules = Robots.parse('User-agent: *\nDisallow: /x$')
expect(Robots.isPathAllowed(rules, '/x', UA)).toBe(false)
expect(Robots.isPathAllowed(rules, '/x/y', UA)).toBe(true)
})
test('a named group beats the wildcard group even when listed later', () => {
const rules = Robots.parse([
'User-agent: *',
'Disallow: /',
'',
`User-agent: ${UA}`,
'Disallow:',
].join('\n'))
expect(Robots.isPathAllowed(rules, '/anything', UA)).toBe(true)
expect(Robots.isPathAllowed(rules, '/anything', 'SomeOtherBot')).toBe(false)
})
})
describe('Robots.isAllowed over the network', () => {
test('refuses a disallowed path', async () => {
serveRobots('User-agent: *\nDisallow: /secret')
await expect(Robots.isAllowed('https://example.com/secret/page')).resolves.toBe(false)
})
test('permits an allowed path', async () => {
serveRobots('User-agent: *\nDisallow: /secret')
await expect(Robots.isAllowed('https://example.com/blog/post')).resolves.toBe(true)
})
// Fail-open: a missing robots.txt is not a refusal.
test('allows everything when robots.txt 404s', async () => {
FetchMock.route('https://example.com/robots.txt', () => FetchMock.status(404))
await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
})
test('allows everything when robots.txt cannot be reached', async () => {
FetchMock.route('https://example.com/robots.txt', FetchMock.networkError())
await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
})
test('caches per origin - two queries, one network call', async () => {
serveRobots('User-agent: *\nDisallow: /secret')
await Robots.isAllowed('https://example.com/a')
await Robots.isAllowed('https://example.com/secret/b')
expect(FetchMock.callsTo('https://example.com/robots.txt')).toHaveLength(1)
})
test('keeps origins separate', async () => {
serveRobots('User-agent: *\nDisallow: /', 'https://blocked.com')
serveRobots('User-agent: *\nDisallow:', 'https://open.com')
await expect(Robots.isAllowed('https://blocked.com/x')).resolves.toBe(false)
await expect(Robots.isAllowed('https://open.com/x')).resolves.toBe(true)
})
test('returns false for an unparseable url rather than throwing', async () => {
await expect(Robots.isAllowed('not a url')).resolves.toBe(false)
})
})
describe('Robots.crawlDelayMs', () => {
test('reports the declared delay', async () => {
serveRobots('User-agent: *\nCrawl-delay: 5')
await expect(Robots.crawlDelayMs('https://example.com/x')).resolves.toBe(5000)
})
test('is null when none is declared', async () => {
serveRobots('User-agent: *\nDisallow: /x')
await expect(Robots.crawlDelayMs('https://example.com/y')).resolves.toBeNull()
})
test('flags a delay beyond the cap instead of parking the job forever', async () => {
expect(Robots.exceedsCrawlDelayCap(3600 * 1000)).toBe(true)
expect(Robots.exceedsCrawlDelayCap(2000)).toBe(false)
expect(Robots.exceedsCrawlDelayCap(null)).toBe(false)
})
})
server/tests/helpers/Schema.test.js
+243
View File
@@ -0,0 +1,243 @@
require('module-alias/register')
const Schema = require('@helpers/Schema')
const db = require('@config/database')
afterAll(() => {
db.end()
})
afterEach(() => {
Schema.__reset()
})
describe('verifyScrapeSchema against the real dev schema', () => {
test('reports ok once the migration has been applied', async () => {
const result = await Schema.verifyScrapeSchema()
if (!result.ok) {
throw new Error(
`dev scrape schema not applied - run ./migrateDatabase.sh. ` +
`missing objects: ${result.missingObjects.join(', ') || 'none'}; ` +
`missing privileges: ${result.missingPrivileges.join(', ') || 'none'}`
)
}
expect(result.missingObjects).toEqual([])
expect(result.missingPrivileges).toEqual([])
expect(result.remedy).toBe('')
})
test('isScrapeSchemaReady reflects the last verification', async () => {
expect(Schema.isScrapeSchemaReady()).toBe(false)
await Schema.verifyScrapeSchema()
expect(Schema.isScrapeSchemaReady()).toBe(true)
Schema.__reset()
expect(Schema.isScrapeSchemaReady()).toBe(false)
})
test('__setReady is honoured, so request paths can be driven in tests', () => {
Schema.__setReady(true)
expect(Schema.isScrapeSchemaReady()).toBe(true)
Schema.__setReady(false)
expect(Schema.isScrapeSchemaReady()).toBe(false)
})
})
//
// The FULLTEXT index is the one object whose loss is silent: MATCH..AGAINST
// against a BTREE index returns nothing without erroring. Guard it by type.
//
describe('FULLTEXT index guard', () => {
test('attachment_text_index exists and is actually FULLTEXT', async () => {
await expect(Schema.__fulltextIndexExists('attachment', 'attachment_text_index'))
.resolves.toBe(true)
})
test('a BTREE index of the same shape would NOT satisfy the check', async () => {
// attachment_scrape_state is a real BTREE index, so this proves the
// check discriminates on type rather than just on name existing.
await expect(Schema.__indexExists('attachment', 'attachment_scrape_state'))
.resolves.toBe(true)
await expect(Schema.__fulltextIndexExists('attachment', 'attachment_scrape_state'))
.resolves.toBe(false)
})
test('verifyScrapeSchema includes it among the verified objects', async () => {
const result = await Schema.verifyScrapeSchema()
expect(result.ok).toBe(true)
expect(result.missingObjects).toEqual([])
})
test('MATCH..AGAINST actually works, not just that an index exists', async () => {
// The functional end of the guard. An index of the right type that
// somehow could not be searched would still be a broken search.
const [rows] = await db.promise().query(
`SELECT COUNT(*) AS n FROM attachment
WHERE MATCH(text) AGAINST(? IN NATURAL LANGUAGE MODE)`,
['the']
)
// Any result (including 0 matches) proves the query planner accepted
// MATCH..AGAINST against this column, which a BTREE index would reject.
expect(typeof rows[0].n).toBe('number')
})
})
describe('object existence checks', () => {
test('finds the real table, columns and index', async () => {
await expect(Schema.__tableExists('scrape_cache')).resolves.toBe(true)
await expect(Schema.__columnExists('attachment', 'scrape_state')).resolves.toBe(true)
await expect(Schema.__columnExists('attachment', 'scrape_attempts')).resolves.toBe(true)
await expect(Schema.__columnExists('attachment', 'enqueued_at')).resolves.toBe(true)
await expect(Schema.__indexExists('attachment', 'attachment_scrape_state')).resolves.toBe(true)
})
test('reports absence rather than throwing for things that do not exist', async () => {
await expect(Schema.__tableExists('no_such_table_xyz')).resolves.toBe(false)
await expect(Schema.__columnExists('attachment', 'no_such_column_xyz')).resolves.toBe(false)
await expect(Schema.__indexExists('attachment', 'no_such_index_xyz')).resolves.toBe(false)
})
})
//
// The privilege check is what turns a future narrow-grants problem into one
// clear boot message instead of a swallowed runtime failure.
//
describe('parseGrants', () => {
const DB = 'application'
const TABLES = ['attachment', 'scrape_cache']
const parse = (lines) => Schema.parseGrants(lines, DB, TABLES)
test('ALL PRIVILEGES ON *.* grants everything', () => {
const granted = parse(['GRANT ALL PRIVILEGES ON *.* TO `root`@`localhost` WITH GRANT OPTION'])
expect(granted.has('SELECT')).toBe(true)
expect(granted.has('INSERT')).toBe(true)
expect(granted.has('UPDATE')).toBe(true)
expect(granted.has('DELETE')).toBe(true)
})
test('ALL PRIVILEGES scoped to the database grants everything', () => {
const granted = parse(['GRANT ALL PRIVILEGES ON `application`.* TO `app`@`localhost`'])
expect(granted.has('DELETE')).toBe(true)
})
test('reads an explicit privilege list', () => {
const granted = parse([
'GRANT USAGE ON *.* TO `app`@`localhost`',
'GRANT SELECT, INSERT, UPDATE, DELETE ON `application`.* TO `app`@`localhost`',
])
expect([...granted].sort()).toEqual(['DELETE', 'INSERT', 'SELECT', 'UPDATE'])
})
// The case that caused this whole change class.
test('spots a missing DELETE', () => {
const granted = parse(['GRANT SELECT, INSERT, UPDATE ON `application`.* TO `app`@`localhost`'])
expect(granted.has('UPDATE')).toBe(true)
expect(granted.has('DELETE')).toBe(false)
})
test('accepts table-scoped grants for our tables', () => {
const granted = parse([
'GRANT SELECT, DELETE ON `application`.`attachment` TO `app`@`localhost`',
'GRANT INSERT, UPDATE ON `application`.`scrape_cache` TO `app`@`localhost`',
])
expect([...granted].sort()).toEqual(['DELETE', 'INSERT', 'SELECT', 'UPDATE'])
})
test('ignores grants on an unrelated database', () => {
const granted = parse(['GRANT ALL PRIVILEGES ON `somethingelse`.* TO `app`@`localhost`'])
expect(granted.size).toBe(0)
})
test('ignores USAGE, which conveys no privileges', () => {
const granted = parse(['GRANT USAGE ON *.* TO `app`@`localhost`'])
expect(granted.size).toBe(0)
})
test('handles grants written without backticks', () => {
const granted = parse(['GRANT SELECT, DELETE ON application.* TO app@localhost'])
expect(granted.has('DELETE')).toBe(true)
})
test('strips column-level qualifiers', () => {
const granted = parse(['GRANT SELECT (id, url), UPDATE (text) ON `application`.`attachment` TO `app`@`localhost`'])
expect(granted.has('SELECT')).toBe(true)
expect(granted.has('UPDATE')).toBe(true)
})
test('ignores lines that are not grant statements', () => {
expect(parse(['', 'not a grant line', null]).size).toBe(0)
})
})
//
// The regression that matters most.
//
// Schema.js used to issue CREATE TABLE / ALTER TABLE through the application's
// own connection pool, which failed on production's least-privilege user with
// "CREATE command denied to user 'app'@'localhost'". Nothing in this module may
// ever issue DDL again.
//
describe('issues no DDL', () => {
test('verifyScrapeSchema runs only read-only statements', async () => {
const statements = []
// db.promise() hands back a NEW wrapper on every call, so spying on one
// instance would miss the one Schema.js obtains internally. Intercept
// the factory and instrument each wrapper it returns.
const realPromise = db.promise.bind(db)
const spy = jest.spyOn(db, 'promise').mockImplementation(() => {
const wrapper = realPromise()
const originalQuery = wrapper.query.bind(wrapper)
wrapper.query = (sql, params) => {
statements.push(String(sql))
return originalQuery(sql, params)
}
return wrapper
})
await Schema.verifyScrapeSchema()
spy.mockRestore()
expect(statements.length).toBeGreaterThan(0)
const forbidden = /\b(CREATE|ALTER|DROP|TRUNCATE|RENAME)\b/i
const offenders = statements.filter(statement => forbidden.test(statement))
expect(offenders).toEqual([])
})
test('the module source contains no DDL outside comments', () => {
const fs = require('fs')
const path = require('path')
const source = fs.readFileSync(
path.join(__dirname, '..', '..', 'helpers', 'Schema.js'), 'utf8'
)
const code = source
.split('\n')
.filter(line => !line.trim().startsWith('//'))
.join('\n')
expect(code).not.toMatch(/\bCREATE\s+TABLE\b/i)
expect(code).not.toMatch(/\bALTER\s+TABLE\b/i)
expect(code).not.toMatch(/\bCREATE\s+INDEX\b/i)
expect(code).not.toMatch(/\bDROP\b/i)
})
})
server/tests/helpers/SiteScrape.test.js
+318
View File
@@ -0,0 +1,318 @@
require('module-alias/register')
const fs = require('fs')
const path = require('path')
const cheerio = require('cheerio')
const SiteScrape = require('@helpers/SiteScrape')
const FIXTURES = path.join(__dirname, '..', 'fixtures', 'html')
// Fixtures are frozen snapshots - see fixtures/html/README.md. No network here.
const load = (name) => cheerio.load(fs.readFileSync(path.join(FIXTURES, name), 'utf8'))
const PAGE_URL = 'https://example.com/blog/2026/post'
describe('getTitle', () => {
test('prefers og:title over the title element', () => {
expect(SiteScrape.getTitle(load('og-complete.html'))).toBe('The Real Title')
})
test('falls back to twitter:title', () => {
expect(SiteScrape.getTitle(load('twitter-only.html'))).toBe('Twitter Card Title')
})
test('falls back to JSON-LD headline', () => {
expect(SiteScrape.getTitle(load('jsonld-article.html'))).toBe('JSON-LD Headline')
})
test('falls back to the title element and collapses whitespace', () => {
expect(SiteScrape.getTitle(load('title-only.html'))).toBe('Just A Title')
})
test('falls back to an h1 when there is no title element', () => {
expect(SiteScrape.getTitle(load('h1-fallback.html'))).toBe('Heading Is All We Have')
})
test('returns empty string for a page with nothing', () => {
expect(SiteScrape.getTitle(load('no-metadata.html'))).toBe('')
})
})
describe('getDescription', () => {
test('prefers og:description', () => {
expect(SiteScrape.getDescription(load('og-complete.html')))
.toBe('A proper social description of the page.')
})
test('falls back to twitter:description', () => {
expect(SiteScrape.getDescription(load('twitter-only.html'))).toBe('Twitter card description.')
})
test('falls back to JSON-LD description', () => {
expect(SiteScrape.getDescription(load('jsonld-article.html'))).toBe('Described in structured data.')
})
test('is empty when no description exists', () => {
expect(SiteScrape.getDescription(load('title-only.html'))).toBe('')
})
})
describe('getJsonLd', () => {
test('reads a plain block', () => {
expect(SiteScrape.getJsonLd(load('jsonld-article.html')).headline).toBe('JSON-LD Headline')
})
test('digs the content node out of an @graph', () => {
expect(SiteScrape.getJsonLd(load('jsonld-graph.html')).headline).toBe('Graph Headline')
})
// Malformed JSON-LD is extremely common and must never throw.
test('survives malformed JSON without throwing', () => {
expect(() => SiteScrape.getJsonLd(load('jsonld-malformed.html'))).not.toThrow()
expect(SiteScrape.getTitle(load('jsonld-malformed.html'))).toBe('Survived Bad JSON')
})
test('is null when absent', () => {
expect(SiteScrape.getJsonLd(load('title-only.html'))).toBeNull()
})
})
describe('getCanonicalUrl', () => {
test('uses link rel=canonical', () => {
expect(SiteScrape.getCanonicalUrl(load('og-complete.html'), PAGE_URL))
.toBe('https://example.com/canonical-post')
})
test('falls back to the requested url', () => {
expect(SiteScrape.getCanonicalUrl(load('title-only.html'), PAGE_URL)).toBe(PAGE_URL)
})
})
describe('getDisplayImage', () => {
test('prefers og:image', () => {
expect(SiteScrape.getDisplayImage(load('og-complete.html'), PAGE_URL))
.toBe('https://cdn.example.com/hero.jpg')
})
test('falls back to twitter:image', () => {
expect(SiteScrape.getDisplayImage(load('twitter-only.html'), PAGE_URL))
.toBe('https://cdn.example.com/twitter.png')
})
test('falls back to the JSON-LD image, taking the first of an array', () => {
expect(SiteScrape.getDisplayImage(load('jsonld-article.html'), PAGE_URL))
.toBe('https://cdn.example.com/ld-primary.jpg')
})
test('reads an object-form JSON-LD image', () => {
expect(SiteScrape.getDisplayImage(load('jsonld-graph.html'), PAGE_URL))
.toBe('https://cdn.example.com/graph.png')
})
// Resolution against the canonical base, not string concatenation.
test('resolves a relative path against the canonical url', () => {
expect(SiteScrape.getDisplayImage(load('relative-images.html'), PAGE_URL))
.toBe('https://example.com/blog/images/hero.png')
})
test('resolves a root-relative icon', () => {
expect(SiteScrape.getDisplayImage(load('root-relative-icon.html'), PAGE_URL))
.toBe('https://example.com/favicon.ico')
})
test('resolves a protocol-relative url', () => {
expect(SiteScrape.getDisplayImage(load('protocol-relative-image.html'), PAGE_URL))
.toBe('https://cdn.example.com/proto.jpg')
})
// The old substring filter dropped all of these.
test('accepts .webp', () => {
expect(SiteScrape.getDisplayImage(load('webp-og-image.html'), PAGE_URL))
.toBe('https://cdn.example.com/hero.webp')
})
test('accepts an extensionless og:image endpoint', () => {
expect(SiteScrape.getDisplayImage(load('extensionless-og-image.html'), PAGE_URL))
.toBe('https://example.com/api/og?title=Hello&theme=dark')
})
test('rejects svg while allowSvgThumbnails is off', () => {
expect(SiteScrape.getDisplayImage(load('svg-only-image.html'), PAGE_URL)).toBe('')
})
test('picks the largest srcset candidate', () => {
expect(SiteScrape.getDisplayImage(load('srcset.html'), PAGE_URL))
.toBe('https://cdn.example.com/large.jpg')
})
test('prefers the largest inline image by declared area', () => {
expect(SiteScrape.getDisplayImage(load('sized-images.html'), PAGE_URL))
.toBe('https://cdn.example.com/big.jpg')
})
test('skips tracking pixels and spacers', () => {
expect(SiteScrape.getDisplayImage(load('tracking-pixels.html'), PAGE_URL))
.toBe('https://cdn.example.com/real-photo.jpg')
})
// Regression: the old code read element.attribs.src directly and threw.
test('does not throw on img tags with no src, and finds a data-src', () => {
const $ = load('imgs-missing-src.html')
expect(() => SiteScrape.getDisplayImage($, PAGE_URL)).not.toThrow()
expect(SiteScrape.getDisplayImage($, PAGE_URL)).toBe('https://cdn.example.com/lazy.jpg')
})
test('prefers a large apple-touch-icon over a 16x16 favicon', () => {
expect(SiteScrape.getDisplayImage(load('apple-touch-icon.html'), PAGE_URL))
.toBe('https://example.com/touch-180.png')
})
test('returns empty string when there is no usable image', () => {
expect(SiteScrape.getDisplayImage(load('no-metadata.html'), PAGE_URL)).toBe('')
})
test('survives malformed markup', () => {
expect(() => SiteScrape.getDisplayImage(load('malformed-unclosed.html'), PAGE_URL)).not.toThrow()
})
})
describe('getImageCandidates', () => {
test('returns an ordered, deduplicated list', () => {
const candidates = SiteScrape.getImageCandidates(load('tracking-pixels.html'), PAGE_URL)
expect(candidates).toEqual(['https://cdn.example.com/real-photo.jpg'])
})
test('is empty rather than null for a bare page', () => {
expect(SiteScrape.getImageCandidates(load('no-metadata.html'), PAGE_URL)).toEqual([])
})
})
describe('getKeywords', () => {
test('extracts page terms and ignores nav, script and footer chrome', () => {
const keywords = SiteScrape.getKeywords(load('keyword-content.html'))
expect(keywords).toMatch(/^Keywords: /)
expect(keywords).toMatch(/beekeeping|beehive|honey/)
expect(keywords).not.toMatch(/navigation/)
expect(keywords).not.toMatch(/tracking/)
expect(keywords).not.toMatch(/footer/)
})
test('returns empty string when there is nothing to count', () => {
expect(SiteScrape.getKeywords(load('no-metadata.html'))).toBe('')
})
test('does not log page content', () => {
// The old implementation dumped up to 6000 chars of every page.
const spy = jest.spyOn(console, 'log').mockImplementation(() => {})
SiteScrape.getKeywords(load('keyword-content.html'))
expect(spy).not.toHaveBeenCalled()
spy.mockRestore()
})
})
describe('getHostName', () => {
test('returns the https origin', () => {
expect(SiteScrape.getHostName('http://www.google.com/search?q=x')).toBe('https://www.google.com')
})
test('returns empty string for a bad url instead of throwing', () => {
expect(() => SiteScrape.getHostName('nonsense')).not.toThrow()
expect(SiteScrape.getHostName('nonsense')).toBe('')
})
})
describe('buildSearchText', () => {
test('joins the parts that exist', () => {
const text = SiteScrape.buildSearchText({
title: 'T', description: 'D', siteName: 'S', keywords: 'Keywords: a, b',
})
expect(text).toBe('T\nD\nS\nKeywords: a, b')
})
test('skips empty parts', () => {
expect(SiteScrape.buildSearchText({ title: 'T', description: '', siteName: '', keywords: '' })).toBe('T')
})
test('caps the length so it cannot bloat attachment.text', () => {
const text = SiteScrape.buildSearchText({ title: 'x'.repeat(50000) })
expect(text.length).toBeLessThanOrEqual(8000)
})
// Sentinel strings used to land in attachment.text and pollute FULLTEXT.
test('never produces the old failure sentinels', () => {
const text = SiteScrape.buildSearchText({ title: '', description: '', siteName: '', keywords: '' })
expect(text).toBe('')
expect(text).not.toContain('No site text')
expect(text).not.toContain('Request Timeout')
})
})
describe('parse', () => {
test('returns everything a link preview needs in one call', () => {
const result = SiteScrape.parse(load('og-complete.html'), PAGE_URL)
expect(result).toMatchObject({
title: 'The Real Title',
description: 'A proper social description of the page.',
siteName: 'Example Journal',
canonicalUrl: 'https://example.com/canonical-post',
imageUrl: 'https://cdn.example.com/hero.jpg',
hostname: 'https://example.com',
})
expect(result.searchText).toContain('The Real Title')
expect(result.searchText).toContain('A proper social description')
})
test('degrades to empty fields rather than throwing on a bare page', () => {
const result = SiteScrape.parse(load('no-metadata.html'), PAGE_URL)
expect(result.title).toBe('')
expect(result.imageUrl).toBe('')
expect(result.searchText).toBe('example.com')
})
})
//
// getCleanUrls is deliberately unchanged - it decides which URLs are detected in
// note text and what string is stored in attachment.url. These tests pin the
// CURRENT behaviour so the rewrite around it cannot drift link detection.
//
describe('getCleanUrls (pinned, must not change)', () => {
test('finds bare and full urls', () => {
expect(SiteScrape.getCleanUrls('see https://example.com/a and www.foo.com here'))
.toEqual(['https://example.com/a', 'https://www.foo.com'])
})
test('still forces http to https', () => {
expect(SiteScrape.getCleanUrls('http://example.com/x')).toEqual(['https://example.com/x'])
})
test('still prepends https to a bare www host', () => {
expect(SiteScrape.getCleanUrls('www.example.com')).toEqual(['https://www.example.com'])
})
test('deduplicates', () => {
expect(SiteScrape.getCleanUrls('https://a.com https://a.com')).toEqual(['https://a.com'])
})
test('returns an empty array when there are no urls', () => {
expect(SiteScrape.getCleanUrls('just some words')).toEqual([])
})
test('returns an empty array for empty text', () => {
expect(SiteScrape.getCleanUrls('')).toEqual([])
})
})
server/tests/helpers/UrlSafety.test.js
+189
View File
@@ -0,0 +1,189 @@
require('module-alias/register')
const UrlSafety = require('@helpers/UrlSafety')
describe('UrlSafety.isPrivateIp', () => {
const privateAddresses = [
'0.0.0.0', '10.0.0.1', '10.255.255.254', '127.0.0.1', '127.1.1.1',
'169.254.1.1', '169.254.169.254', // cloud metadata endpoint
'172.16.0.1', '172.31.255.254', '192.168.1.1', '192.0.0.1', '192.0.2.5',
'198.18.0.1', '100.64.0.1', '100.127.255.254', '224.0.0.1', '255.255.255.255',
'::1', '::', 'fc00::1', 'fd12:3456::1', 'fe80::1', 'ff02::1',
'::ffff:127.0.0.1', '::ffff:10.0.0.1', '64:ff9b::127.0.0.1',
]
privateAddresses.forEach(address => {
test(`blocks ${address}`, () => {
expect(UrlSafety.isPrivateIp(address)).toBe(true)
})
})
const publicAddresses = ['1.1.1.1', '8.8.8.8', '93.184.216.34', '172.32.0.1', '2606:4700::1111']
publicAddresses.forEach(address => {
test(`allows ${address}`, () => {
expect(UrlSafety.isPrivateIp(address)).toBe(false)
})
})
test('treats a non-IP string as unsafe', () => {
expect(UrlSafety.isPrivateIp('not-an-ip')).toBe(true)
expect(UrlSafety.isPrivateIp('')).toBe(true)
})
})
describe('UrlSafety.parse', () => {
test('rejects non-http schemes', () => {
expect(UrlSafety.parse('file:///etc/passwd')).toBeNull()
expect(UrlSafety.parse('ftp://example.com/x')).toBeNull()
expect(UrlSafety.parse('javascript:alert(1)')).toBeNull()
expect(UrlSafety.parse('data:text/html,<h1>x</h1>')).toBeNull()
})
test('rejects garbage and empty input', () => {
expect(UrlSafety.parse('')).toBeNull()
expect(UrlSafety.parse(null)).toBeNull()
expect(UrlSafety.parse('not a url at all')).toBeNull()
})
test('rejects URLs longer than the attachment.url column', () => {
const long = 'https://example.com/' + 'a'.repeat(1200)
expect(UrlSafety.parse(long)).toBeNull()
})
test('accepts ordinary http and https', () => {
expect(UrlSafety.parse('https://example.com/a').href).toBe('https://example.com/a')
expect(UrlSafety.parse('http://example.com/a').href).toBe('http://example.com/a')
})
})
describe('UrlSafety.normalizeForKey / urlKey', () => {
test('strips tracking params so the cache key is stable', () => {
const bare = UrlSafety.urlKey('https://example.com/post')
const noisy = UrlSafety.urlKey('https://example.com/post?utm_source=x&utm_medium=y&fbclid=z')
expect(noisy).toBe(bare)
})
test('keeps meaningful query params', () => {
const a = UrlSafety.urlKey('https://example.com/search?q=cats')
const b = UrlSafety.urlKey('https://example.com/search?q=dogs')
expect(a).not.toBe(b)
})
test('is insensitive to param order, host case, fragment and trailing host dot', () => {
const canonical = UrlSafety.urlKey('https://example.com/p?a=1&b=2')
expect(UrlSafety.urlKey('https://example.com/p?b=2&a=1')).toBe(canonical)
expect(UrlSafety.urlKey('https://EXAMPLE.com/p?a=1&b=2')).toBe(canonical)
expect(UrlSafety.urlKey('https://example.com/p?a=1&b=2#section')).toBe(canonical)
expect(UrlSafety.urlKey('https://example.com./p?a=1&b=2')).toBe(canonical)
})
test('distinguishes http from https', () => {
expect(UrlSafety.urlKey('http://example.com/')).not.toBe(UrlSafety.urlKey('https://example.com/'))
})
test('returns a 64-char hex digest', () => {
expect(UrlSafety.urlKey('https://example.com/')).toMatch(/^[0-9a-f]{64}$/)
})
test('returns null for an unusable URL', () => {
expect(UrlSafety.urlKey('javascript:alert(1)')).toBeNull()
})
})
describe('UrlSafety.sameResource', () => {
// This is what stops scanTextForWebsites deleting and re-scraping a row
// every time a note is saved.
test('treats http/https, www. and trailing slash as the same resource', () => {
expect(UrlSafety.sameResource('http://foo.com/a', 'https://foo.com/a')).toBe(true)
expect(UrlSafety.sameResource('https://www.foo.com/a', 'https://foo.com/a')).toBe(true)
expect(UrlSafety.sameResource('https://foo.com/a/', 'https://foo.com/a')).toBe(true)
expect(UrlSafety.sameResource('https://foo.com', 'https://foo.com/')).toBe(true)
})
test('does not conflate different paths, hosts or queries', () => {
expect(UrlSafety.sameResource('https://foo.com/a', 'https://foo.com/b')).toBe(false)
expect(UrlSafety.sameResource('https://foo.com/a', 'https://bar.com/a')).toBe(false)
expect(UrlSafety.sameResource('https://foo.com/a?x=1', 'https://foo.com/a?x=2')).toBe(false)
})
test('is false when either side is unusable', () => {
expect(UrlSafety.sameResource('https://foo.com/a', 'javascript:x')).toBe(false)
})
})
describe('UrlSafety.assertFetchable', () => {
const expectRejection = async (url, reason) => {
await expect(UrlSafety.assertFetchable(url)).rejects.toMatchObject({ name: 'SafetyError', reason })
}
test('refuses non-http schemes', async () => {
await expectRejection('file:///etc/passwd', 'bad_url')
})
test('refuses embedded credentials', async () => {
await expectRejection('https://user:[email protected]/', 'credentials_in_url')
})
test('refuses localhost and internal suffixes', async () => {
await expectRejection('http://localhost:3000/', 'blocked_host')
await expectRejection('http://db.internal/', 'blocked_host')
await expectRejection('http://thing.local/', 'blocked_host')
await expectRejection('http://box.lan/', 'blocked_host')
})
test('refuses literal private addresses without touching DNS', async () => {
await expectRejection('http://127.0.0.1:3000/', 'private_ip')
await expectRejection('http://169.254.169.254/latest/meta-data/', 'private_ip')
await expectRejection('http://10.1.2.3/', 'private_ip')
await expectRejection('http://[::1]:8080/', 'private_ip')
})
test('allows a literal public address', async () => {
await expect(UrlSafety.assertFetchable('http://1.1.1.1/')).resolves.toEqual(['1.1.1.1'])
})
test('refuses a hostname that resolves to a private address', async () => {
const dns = require('dns').promises
const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '192.168.0.5', family: 4 }])
await expectRejection('https://rebind.example.com/', 'private_ip')
spy.mockRestore()
})
test('refuses when ANY address in a round-robin record is private', async () => {
const dns = require('dns').promises
const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([
{ address: '93.184.216.34', family: 4 },
{ address: '127.0.0.1', family: 4 },
])
await expectRejection('https://mixed.example.com/', 'private_ip')
spy.mockRestore()
})
test('allows a hostname that resolves entirely to public addresses', async () => {
const dns = require('dns').promises
const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
await expect(UrlSafety.assertFetchable('https://example.com/')).resolves.toEqual(['93.184.216.34'])
spy.mockRestore()
})
test('surfaces a DNS failure as dns_failed', async () => {
const dns = require('dns').promises
const spy = jest.spyOn(dns, 'lookup').mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
await expectRejection('https://nonexistent.example/', 'dns_failed')
spy.mockRestore()
})
})
server/tests/models/Attachment.test.js
+122 -9
View File
@@ -1,5 +1,12 @@
const Attachment = require('../../models/Attachment')
const User = require('../../models/User')
const ScrapeQueue = require('@workers/ScrapeQueue')
const Schema = require('@helpers/Schema')
const Robots = require('@helpers/Robots')
const FetchMock = require('../support/fetchMock')
const dns = require('dns').promises
let dnsSpy = null
const testUserName = 'jestTestUserAttachment'
const password = 'Beans19934!!!'
@@ -67,19 +74,125 @@ test('Test get Push Key Bookmarklet', () => {
})
test('Test Push URL', () => {
//
// These used to hit https://www.solidscribe.com over the live network, so the
// suite needed internet access and the assertion depended on whatever that page
// happened to serve. fetch is mocked against a fixture instead.
//
describe('Push URL', () => {
let url = 'https://www.solidscribe.com'
beforeEach(() => {
FetchMock.install()
// The server marks this at boot; there is no boot phase in tests.
Schema.__setReady(true)
Robots.__resetCache()
dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
// Real code path, no politeness waiting and no robots lookups.
ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false })
})
afterEach(() => {
FetchMock.restore()
dnsSpy.mockRestore()
ScrapeQueue.__reset()
Schema.__reset()
})
test('scrapes a pushed url into a searchable attachment', () => {
const url = 'https://www.solidscribe.com/'
FetchMock.route(url, () => FetchMock.html(`
<html><head>
<title>Solid Scribe</title>
<meta property="og:description" content="An encrypted note taking application.">
</head><body><p>x</p></body></html>
`))
return Attachment.pushUrl(newPushKey, url)
.then(( results => {
return Attachment.textSearch(newUserId, 'scribe')
}))
.then((results) => {
.then(() => ScrapeQueue.drain())
.then(() => Attachment.textSearch(newUserId, 'scribe'))
.then(results => {
expect(results.length == 1).toBe(true)
expect(results[0].text).toContain('Solid Scribe')
})
})
test('stores the description so it is searchable too', () => {
const url = 'https://www.solidscribe.com/encrypted'
FetchMock.route(url, () => FetchMock.html(`
<html><head>
<title>Encryption Page</title>
<meta property="og:description" content="Notes are sealed with aes256gcm.">
</head><body><p>x</p></body></html>
`))
return Attachment.pushUrl(newPushKey, url)
.then(() => ScrapeQueue.drain())
.then(() => Attachment.textSearch(newUserId, 'aes256gcm'))
.then(results => {
expect(results.length).toBeGreaterThan(0)
})
})
// The pushmebaby query param was previously passed through with no
// validation whatsoever.
test('refuses a pushed url pointing at loopback and makes no request', () => {
FetchMock.route(/.*/, () => FetchMock.html('<title>should never load</title>'))
return Attachment.pushUrl(newPushKey, 'http://127.0.0.1:3000/admin')
.then(result => {
expect(result).toBe(true)
expect(FetchMock.callCount()).toBe(0)
})
})
test('refuses a pushed url pointing at the cloud metadata endpoint', () => {
FetchMock.route(/.*/, () => FetchMock.html('<title>should never load</title>'))
return Attachment.pushUrl(newPushKey, 'http://169.254.169.254/latest/meta-data/')
.then(() => {
expect(FetchMock.callCount()).toBe(0)
})
})
test('resolves truthy for an unknown pushkey without scraping', () => {
FetchMock.route(/.*/, () => FetchMock.html('<title>nope</title>'))
return Attachment.pushUrl('not-a-real-pushkey', 'https://example.com/')
.then(result => {
expect(result).toBe(true)
expect(FetchMock.callCount()).toBe(0)
})
})
// Previously this could hang the HTTP response indefinitely.
test('returns within the budget when the server never responds', () => {
const url = 'https://www.solidscribe.com/slow'
FetchMock.route(url, () => FetchMock.hang())
ScrapeQueue.configure({
perHostMinSpacingMs: 0,
perHostMaxSpacingMs: 0,
checkRobots: false,
pageTimeoutMs: 80,
totalBudgetMs: 300,
retries: 0,
})
const startedAt = Date.now()
return Attachment.pushUrl(newPushKey, url)
.then(result => {
expect(result).toBe(true)
expect(Date.now() - startedAt).toBeLessThan(5000)
})
})
})
server/tests/models/ScrapeQueue.test.js
+455
View File
@@ -0,0 +1,455 @@
require('module-alias/register')
const db = require('@config/database')
const User = require('@models/User')
const Attachment = require('@models/Attachment')
const ScrapeCache = require('@models/ScrapeCache')
const ScrapeQueue = require('@workers/ScrapeQueue')
const Schema = require('@helpers/Schema')
const Robots = require('@helpers/Robots')
const HttpFetch = require('@helpers/HttpFetch')
const UrlSafety = require('@helpers/UrlSafety')
const FetchMock = require('../support/fetchMock')
const dns = require('dns').promises
const testUserName = 'jestTestUserScrapeQueue'
const password = 'Beans19934!!!'
let userId = null
let noteId = null
let dnsSpy = null
const page = (title, description) => FetchMock.html(`
<html><head>
<title>${title}</title>
<meta property="og:description" content="${description || 'a description'}">
</head><body><p>body</p></body></html>
`)
// Clear cache rows for a URL so each test starts from a real miss.
const clearCache = (url) => db.promise()
.query('DELETE FROM scrape_cache WHERE url_hash = ?', [UrlSafety.urlKey(url)])
const attachmentRow = (id) => db.promise()
.query('SELECT * FROM attachment WHERE id = ?', [id])
.then(([rows]) => rows[0] || null)
beforeAll(async () => {
// The server normally does this at boot; tests have no boot phase.
const verification = await Schema.verifyScrapeSchema()
if (!verification.ok) {
throw new Error(`dev scrape schema not applied - run ./migrateDatabase.sh (${verification.remedy})`)
}
try {
const existing = await User.getByUserName(testUserName)
if (existing && existing.id) { await User.deleteUser(existing.id, password) }
} catch (error) { /* first run */ }
const registered = await User.register(testUserName, password)
userId = registered.userId
// A real note, so the soft-delete path below sees an existing note and
// hides the attachment rather than deleting it.
const [note] = await db.promise().query(
'INSERT INTO note (user_id, created, opened) VALUES (?, ?, ?)',
[userId, Math.round(Date.now() / 1000), Math.round(Date.now() / 1000)]
)
noteId = note.insertId
})
beforeEach(() => {
FetchMock.install()
Schema.__setReady(true)
Robots.__resetCache()
dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
HttpFetch.__setSleep(() => Promise.resolve())
// Exercise the real code path with politeness spacing and robots disabled.
ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false })
})
afterEach(() => {
FetchMock.restore()
dnsSpy.mockRestore()
HttpFetch.__resetSleep()
ScrapeQueue.__reset()
})
afterAll(async () => {
await db.promise().query('DELETE FROM attachment WHERE user_id = ?', [userId])
await db.promise().query('DELETE FROM note WHERE user_id = ?', [userId])
db.end()
})
describe('fetch and persist', () => {
test('writes parsed metadata onto the attachment row as plaintext', async () => {
const url = 'https://example.com/post-one'
await clearCache(url)
FetchMock.route(url, () => page('Post One', 'the first post'))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
const row = await attachmentRow(attachmentId)
expect(row.text).toContain('Post One')
expect(row.text).toContain('the first post')
expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
// Must stay unencrypted - the FULLTEXT index depends on it.
expect(row.text).not.toMatch(/^[0-9a-f]{64}/)
})
test('keeps the url the user wrote rather than the canonical url', async () => {
const url = 'https://example.com/original'
await clearCache(url)
FetchMock.route(url, () => FetchMock.html(`
<html><head><title>Canonical Test</title>
<link rel="canonical" href="https://example.com/different-canonical">
</head><body>x</body></html>
`))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
const row = await attachmentRow(attachmentId)
expect(row.url).toBe(url)
// The canonical URL is recorded in the cache, not on the attachment.
const cached = await ScrapeCache.get(url)
expect(cached.canonical_url).toBe('https://example.com/different-canonical')
})
})
describe('caching', () => {
test('a second request for the same url makes zero network calls', async () => {
const url = 'https://example.com/cached'
await clearCache(url)
FetchMock.route(url, () => page('Cached Page'))
const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
const callsAfterFirst = FetchMock.callCount()
expect(callsAfterFirst).toBeGreaterThan(0)
const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
expect(FetchMock.callCount()).toBe(callsAfterFirst)
// The cached row still populates the second attachment.
const row = await attachmentRow(secondId)
expect(row.text).toContain('Cached Page')
expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
})
test('two notes sharing a url coalesce into one fetch', async () => {
const url = 'https://example.com/shared'
await clearCache(url)
FetchMock.route(url, () => page('Shared Page'))
const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
// Enqueued together, so they overlap in flight.
await ScrapeQueue.enqueueUrls([
{ userId, noteId: null, attachmentId: firstId, url },
{ userId, noteId: null, attachmentId: secondId, url },
])
expect(FetchMock.callsTo(url)).toHaveLength(1)
const rows = await Promise.all([attachmentRow(firstId), attachmentRow(secondId)])
rows.forEach(row => {
expect(row.text).toContain('Shared Page')
expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
})
})
test('a stale entry revalidates and a 304 reuses the stored text', async () => {
const url = 'https://example.com/revalidate'
await clearCache(url)
FetchMock.routeSequence(url, [
FetchMock.html('<html><head><title>Original Title</title></head><body>x</body></html>', {
headers: { etag: 'W/"v1"' },
}),
FetchMock.status(304),
])
const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
// Force staleness so the next pass revalidates instead of hitting cache.
await db.promise().query('UPDATE scrape_cache SET expires_at = ? WHERE url_hash = ?',
[Math.round(Date.now() / 1000) - 10, UrlSafety.urlKey(url)])
const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
// Conditional request was sent.
const conditional = FetchMock.callsTo(url).find(call => call.headers['If-None-Match'])
expect(conditional).toBeTruthy()
const row = await attachmentRow(secondId)
expect(row.text).toContain('Original Title')
})
})
describe('failure handling', () => {
test('a 500 leaves the row failed and never writes a sentinel string', async () => {
const url = 'https://example.com/broken'
await clearCache(url)
FetchMock.route(url, () => FetchMock.status(500))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
const row = await attachmentRow(attachmentId)
expect(row.scrape_state).toBe(ScrapeQueue.STATE.FAILED)
// The old code stored these in attachment.text and fed them to FULLTEXT.
expect(row.text).not.toContain('No site text')
expect(row.text).not.toContain('Request Timeout')
})
test('a 404 removes the shell row rather than orphaning it', async () => {
const url = 'https://example.com/gone'
await clearCache(url)
FetchMock.route(url, () => FetchMock.status(404))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
expect(await attachmentRow(attachmentId)).toBeNull()
})
test('a blocked url removes the shell row and makes no request', async () => {
const url = 'http://127.0.0.1:9999/internal'
FetchMock.route(/.*/, () => page('should never load'))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
expect(await attachmentRow(attachmentId)).toBeNull()
expect(FetchMock.callCount()).toBe(0)
})
test('one failing url does not strand the rest of a batch', async () => {
const good = 'https://example.com/good-one'
const bad = 'https://example.com/bad-one'
await clearCache(good)
await clearCache(bad)
FetchMock.route(good, () => page('Good One'))
FetchMock.route(bad, () => FetchMock.status(500))
const goodId = await Attachment.createPendingUrlAttachment(userId, null, good)
const badId = await Attachment.createPendingUrlAttachment(userId, null, bad)
// The old counter-based implementation never settled in this case.
const texts = await ScrapeQueue.enqueueUrls([
{ userId, noteId: null, attachmentId: badId, url: bad },
{ userId, noteId: null, attachmentId: goodId, url: good },
])
expect(texts).toHaveLength(2)
expect((await attachmentRow(goodId)).text).toContain('Good One')
})
test('a failed url is not refetched while inside its backoff window', async () => {
const url = 'https://example.com/flaky'
await clearCache(url)
FetchMock.route(url, () => FetchMock.status(500))
const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
const callsAfterFirst = FetchMock.callCount()
const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
expect(FetchMock.callCount()).toBe(callsAfterFirst)
})
})
describe('robots.txt', () => {
test('refuses a disallowed url and records it', async () => {
const url = 'https://example.com/no-bots/page'
await clearCache(url)
ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0 }) // robots ON
FetchMock.route('https://example.com/robots.txt', () => new Response(
'User-agent: *\nDisallow: /no-bots', { status: 200, headers: { 'content-type': 'text/plain' } }
))
FetchMock.route(url, () => page('Should Not Be Fetched'))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
expect(FetchMock.callsTo(url)).toHaveLength(0)
const cached = await ScrapeCache.get(url)
expect(cached.fetch_state).toBe('blocked')
})
test('proceeds when robots.txt allows the path', async () => {
const url = 'https://example.com/allowed/page'
await clearCache(url)
ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0 })
FetchMock.route('https://example.com/robots.txt', () => new Response(
'User-agent: *\nDisallow: /private', { status: 200, headers: { 'content-type': 'text/plain' } }
))
FetchMock.route(url, () => page('Allowed Page'))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
expect((await attachmentRow(attachmentId)).text).toContain('Allowed Page')
})
})
//
// Production ran for a while with the scrape columns missing, and every note
// save attempted an INSERT that could not succeed. These pin the degraded
// behaviour: do nothing, quietly, and leave everything else working.
//
describe('degrades cleanly when the scrape schema is absent', () => {
beforeEach(() => {
Schema.__setReady(false)
})
test('scanTextForWebsites creates no rows and makes no requests', async () => {
FetchMock.route(/.*/, () => page('Should Not Be Fetched'))
const [before] = await db.promise().query(
'SELECT COUNT(*) AS n FROM attachment WHERE user_id = ?', [userId])
const text = await Attachment.scanTextForWebsites(userId, null, 'see https://example.com/unavailable')
const [after] = await db.promise().query(
'SELECT COUNT(*) AS n FROM attachment WHERE user_id = ?', [userId])
expect(after[0].n).toBe(before[0].n)
expect(FetchMock.callCount()).toBe(0)
expect(typeof text).toBe('string')
})
//
// Removing a link from a note must still clean up its attachment. That path
// touches none of the new columns, so it is unaffected by the outage.
//
// Note the existing semantics: scanTextForWebsites calls
// Attachment.delete(..., urlDelete = true), which HARD deletes. The
// visible = 0 soft-delete applies only when a user removes an attachment
// directly while its note still exists (urlDelete = false).
//
test('still removes an attachment whose url left the note', async () => {
Schema.__setReady(true)
const url = 'https://example.com/will-be-removed'
await clearCache(url)
FetchMock.route(url, () => page('Removed Later'))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, noteId, url)
await ScrapeQueue.enqueueUrl({ userId, noteId, attachmentId, url })
expect(await attachmentRow(attachmentId)).not.toBeNull()
// Now the schema "disappears" and the note is saved without that link.
Schema.__setReady(false)
await Attachment.scanTextForWebsites(userId, noteId, 'no links here any more')
// Attachment.delete is deliberately fire-and-forget in scanTextForWebsites,
// so give it a tick to land.
await new Promise(resolve => setTimeout(resolve, 150))
expect(await attachmentRow(attachmentId)).toBeNull()
})
test('scrapeUrlsCreateAttachments resolves empty without touching the DB', async () => {
FetchMock.route(/.*/, () => page('nope'))
await expect(Attachment.scrapeUrlsCreateAttachments(userId, null, ['https://example.com/x']))
.resolves.toBe('')
expect(FetchMock.callCount()).toBe(0)
})
test('pushUrl keeps its contract and resolves truthy', async () => {
FetchMock.route(/.*/, () => page('nope'))
await expect(Attachment.pushUrl('any-key', 'https://example.com/x')).resolves.toBe(true)
expect(FetchMock.callCount()).toBe(0)
})
test('recoverPending returns zero without querying', async () => {
await expect(ScrapeQueue.recoverPending()).resolves.toEqual({ requeued: 0, retired: 0 })
})
test('enqueueUrl is a no-op backstop', async () => {
FetchMock.route(/.*/, () => page('nope'))
await expect(ScrapeQueue.enqueueUrl({
userId, noteId: null, attachmentId: null, url: 'https://example.com/x',
})).resolves.toBe('')
expect(FetchMock.callCount()).toBe(0)
})
})
describe('crash recovery', () => {
test('re-enqueues rows left pending by a restart', async () => {
const url = 'https://example.com/recovered'
await clearCache(url)
FetchMock.route(url, () => page('Recovered Page'))
// Simulate a row that was enqueued but never processed.
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
const recovery = await ScrapeQueue.recoverPending()
expect(recovery.requeued).toBeGreaterThan(0)
await ScrapeQueue.drain()
const row = await attachmentRow(attachmentId)
expect(row.text).toContain('Recovered Page')
expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
})
test('resets rows stuck mid-fetch so they are retried', async () => {
const url = 'https://example.com/stuck'
await clearCache(url)
FetchMock.route(url, () => page('Unstuck Page'))
const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
// Pretend the process died while this one was in flight.
await db.promise().query('UPDATE attachment SET scrape_state = ? WHERE id = ?',
[ScrapeQueue.STATE.FETCHING, attachmentId])
await ScrapeQueue.recoverPending()
await ScrapeQueue.drain()
expect((await attachmentRow(attachmentId)).scrape_state).toBe(ScrapeQueue.STATE.DONE)
})
test('does not sweep up long-settled rows', async () => {
// Rows default to DONE, which is what keeps the pre-existing ~1400
// attachment rows out of the recovery queue on first boot.
const [rows] = await db.promise().query(
'SELECT COUNT(*) AS n FROM attachment WHERE scrape_state = ? AND user_id = ?',
[ScrapeQueue.STATE.PENDING, userId]
)
const recovery = await ScrapeQueue.recoverPending()
expect(recovery.requeued).toBeLessThanOrEqual(rows[0].n + 1)
})
})
server/tests/schema/migrations.test.js
+220
View File
@@ -0,0 +1,220 @@
require('module-alias/register')
const { execFileSync } = require('child_process')
const fs = require('fs')
const path = require('path')
const REPO = path.join(__dirname, '..', '..', '..')
//
// These shell out to the Prisma CLI, which only exists inside nix-shell. Skip
// rather than fail when it is absent, so `npx jest` still works outside the
// shell -- but do NOT skip silently in a way that hides real drift when it IS
// available.
//
const hasPrisma = (() => {
try {
execFileSync('prisma', ['--version'], { stdio: 'ignore' })
return true
} catch (error) {
return false
}
})()
const describeWithPrisma = hasPrisma ? describe : describe.skip
if (!hasPrisma) {
console.log('[schema tests] prisma not on PATH - skipping. Run inside nix-shell to exercise these.')
}
const prisma = (args) => execFileSync('prisma', args, {
cwd: REPO,
encoding: 'utf8',
env: process.env,
})
describe('committed migration artifacts', () => {
test('the baseline migration and lock file are present', () => {
expect(fs.existsSync(path.join(REPO, 'prisma/schema.prisma'))).toBe(true)
expect(fs.existsSync(path.join(REPO, 'prisma/migrations/0_init/migration.sql'))).toBe(true)
expect(fs.existsSync(path.join(REPO, 'prisma/migrations/migration_lock.toml'))).toBe(true)
})
//
// Prisma emits COLLATE utf8mb4_unicode_ci for MySQL, but every table in this
// database is utf8mb4_general_ci, and `prisma migrate diff` does NOT report
// the difference. So a regenerated migration would silently build a
// differently-collated database. This asserts the hand-correction is intact.
//
test('no migration declares a collation other than utf8mb4_general_ci', () => {
const dir = path.join(REPO, 'prisma/migrations')
const files = fs.readdirSync(dir, { withFileTypes: true })
.filter(entry => entry.isDirectory())
.map(entry => path.join(dir, entry.name, 'migration.sql'))
.filter(file => fs.existsSync(file))
expect(files.length).toBeGreaterThan(0)
files.forEach(file => {
const ddl = fs.readFileSync(file, 'utf8')
.split('\n')
.filter(line => !line.trim().startsWith('--'))
.join('\n')
const collations = [...ddl.matchAll(/COLLATE\s+([A-Za-z0-9_]+)/gi)].map(match => match[1])
collations.forEach(collation => {
expect(collation).toBe('utf8mb4_general_ci')
})
})
})
test('the baseline creates the FULLTEXT index, not a plain one', () => {
const sql = fs.readFileSync(path.join(REPO, 'prisma/migrations/0_init/migration.sql'), 'utf8')
expect(sql).toMatch(/FULLTEXT INDEX `attachment_text_index`/)
// A plain INDEX of that name would be the silent-failure mode.
expect(sql).not.toMatch(/^\s*INDEX `attachment_text_index`/m)
})
//
// @@fulltext is GA in Prisma 7 -- the `fullTextIndex` preview flag no longer
// exists in the CLI at all (verified: zero occurrences in the 7.8.0 bundle).
// It was silently ignored rather than rejected, which makes it exactly the
// kind of dead config that misleads the next reader.
//
test('the schema declares @@fulltext and no longer carries the dead preview flag', () => {
const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8')
expect(schema).toMatch(/@@fulltext\(\[text\]/)
expect(schema).not.toMatch(/fullTextIndex/)
})
//
// No generator block means `prisma generate` cannot produce a client into
// this repo. That is the structural guarantee behind "migrations only" --
// stronger than a convention, because it cannot be done by accident.
//
test('the schema declares no generator, so no client can be generated', () => {
const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8')
expect(schema).not.toMatch(/generator\s+\w+\s*\{/)
expect(schema).toMatch(/datasource\s+db\s*\{/)
})
//
// The running server must never depend on Prisma.
//
test('no application code imports prisma', () => {
const { execFileSync } = require('child_process')
let hits = ''
try {
hits = execFileSync('grep', [
'-rn', '--include=*.js',
'-E', "require\\(['\"]@?prisma", path.join(REPO, 'server'),
], { encoding: 'utf8' })
} catch (error) {
hits = '' // grep exits 1 when nothing matches
}
expect(hits.trim()).toBe('')
})
test('no @relation is introduced - this database has no foreign keys', () => {
// Adding relations would mean adding FK constraints to a live database,
// which is a schema change well outside adopting a migration tool.
const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8')
expect(schema).not.toMatch(/@relation/)
})
})
describeWithPrisma('schema drift', () => {
//
// The check that keeps schema.prisma honest: if the live database and the
// committed schema ever diverge, this fails.
//
test('schema.prisma matches the live database exactly', () => {
const output = prisma([
'migrate', 'diff',
'--from-config-datasource',
'--to-schema', 'prisma/schema.prisma',
'--script',
])
expect(output).toMatch(/This is an empty migration/)
})
//
// The load-bearing guarantee: the committed migrations, applied to an empty
// database, reproduce the live schema exactly.
//
// Done by actually rebuilding into a scratch database and comparing
// information_schema, rather than via `migrate diff --from-migrations`
// (which requires a shadow database) -- and this is the stronger check
// anyway, because it also catches collation drift, which Prisma's own diff
// does NOT report.
//
test('the committed migrations rebuild the live schema exactly', () => {
const db = require('@config/database')
const SCRATCH = 'prisma_drift_check'
const mysqlBin = (() => {
try { execFileSync('mariadb', ['--version'], { stdio: 'ignore' }); return 'mariadb' }
catch (error) { return 'mysql' }
})()
const adminArgs = ['-u', 'root']
const socket = path.join(REPO, '.mysql', 'mysql.sock')
if (fs.existsSync(socket)) { adminArgs.push(`--socket=${socket}`) }
const admin = (sql, database) => execFileSync(
mysqlBin,
[...adminArgs, '-N', '-B', ...(database ? [database] : []), '-e', sql],
{ encoding: 'utf8' }
)
try {
admin(`DROP DATABASE IF EXISTS \`${SCRATCH}\`; CREATE DATABASE \`${SCRATCH}\`;`)
execFileSync('prisma', ['migrate', 'deploy'], {
cwd: REPO,
encoding: 'utf8',
env: { ...process.env, DATABASE_URL: `mysql://[email protected]:3306/${SCRATCH}` },
})
const snapshot = (schema) => ({
columns: admin(
`SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLLATION_NAME
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations'
ORDER BY TABLE_NAME, COLUMN_NAME`
),
indexes: admin(
`SELECT TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX, COLUMN_NAME, NON_UNIQUE, INDEX_TYPE
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations'
ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX`
),
collations: admin(
`SELECT TABLE_NAME, TABLE_COLLATION FROM information_schema.TABLES
WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations'
ORDER BY TABLE_NAME`
),
})
const live = snapshot('application')
const rebuilt = snapshot(SCRATCH)
expect(rebuilt.columns).toBe(live.columns)
expect(rebuilt.indexes).toBe(live.indexes)
expect(rebuilt.collations).toBe(live.collations)
} finally {
try { admin(`DROP DATABASE IF EXISTS \`${SCRATCH}\`;`) } catch (error) { /* best effort */ }
}
})
test('migrate status reports no pending migrations', () => {
const output = prisma(['migrate', 'status'])
expect(output).toMatch(/Database schema is up to date/)
})
})
server/tests/support/fetchMock.js
+127
View File
@@ -0,0 +1,127 @@
//
// Test harness for the outbound HTTP layer.
//
// Responses are built from the REAL Response/ReadableStream globals rather than
// hand-rolled stubs, so the streaming size cap, the charset decoder and the
// body-cancel paths in HttpFetch are genuinely exercised instead of being
// stubbed around.
//
let FetchMock = module.exports = {}
let originalFetch = null
let routes = []
let callLog = []
FetchMock.install = () => {
originalFetch = global.fetch
routes = []
callLog = []
global.fetch = jest.fn(async (url, options = {}) => {
const href = typeof url === 'string' ? url : url.href
callLog.push({ url: href, headers: options.headers || {}, options })
const route = routes.find(entry => entry.match(href))
if (!route) {
throw Object.assign(new Error(`No route for ${href}`), { code: 'ENOTFOUND' })
}
// A hanging route must still honour the abort signal, or the test would
// wait forever instead of proving the AbortController fires.
if (options.signal) {
return await Promise.race([
Promise.resolve(route.responder(href, options)),
abortPromise(options.signal),
])
}
return route.responder(href, options)
})
}
const abortPromise = (signal) => new Promise((resolve, reject) => {
if (signal.aborted) {
return reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }))
}
signal.addEventListener('abort', () => {
reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }))
}, { once: true })
})
FetchMock.restore = () => {
if (originalFetch) { global.fetch = originalFetch } else { delete global.fetch }
originalFetch = null
routes = []
callLog = []
}
//
// route('https://example.com/', responder) or route(/example/, responder).
// Responders are called with (url, options) and may be functions returning a
// Response, so a route can answer differently on each call.
//
FetchMock.route = (pattern, responder) => {
const match = pattern instanceof RegExp
? (href) => pattern.test(href)
: (href) => href === pattern
routes.push({ match, responder: typeof responder === 'function' ? responder : () => responder })
}
// Answer with a counter so "retried exactly N times" is assertable.
FetchMock.routeSequence = (pattern, responses) => {
let index = 0
FetchMock.route(pattern, () => {
const response = responses[Math.min(index, responses.length - 1)]
index++
return typeof response === 'function' ? response() : response
})
}
FetchMock.html = (body, { status = 200, headers = {}, contentType = 'text/html; charset=utf-8' } = {}) => {
return new Response(body, {
status,
headers: { 'content-type': contentType, ...headers },
})
}
// Raw bytes, for the charset tests where the encoding is the point.
FetchMock.buffer = (buffer, { status = 200, contentType = 'text/html', headers = {} } = {}) => {
return new Response(buffer, {
status,
headers: { 'content-type': contentType, ...headers },
})
}
FetchMock.status = (status, { headers = {}, body = '' } = {}) => {
return new Response(body || null, { status, headers })
}
FetchMock.redirect = (location, status = 302) => {
return new Response(null, { status, headers: { location } })
}
//
// A body that never ends. Proves the size cap aborts mid-stream rather than
// buffering until the process dies.
//
FetchMock.oversized = (contentType = 'text/html') => {
const chunk = Buffer.alloc(64 * 1024, 0x61)
const stream = new ReadableStream({
pull(controller) { controller.enqueue(new Uint8Array(chunk)) },
})
return new Response(stream, { status: 200, headers: { 'content-type': contentType } })
}
// Never settles - the AbortController is the only way out.
FetchMock.hang = () => new Promise(() => {})
FetchMock.networkError = (code = 'ECONNRESET') => () => {
throw Object.assign(new Error(`socket ${code}`), { code })
}
FetchMock.calls = () => callLog.slice()
FetchMock.callsTo = (pattern) => callLog.filter(entry =>
pattern instanceof RegExp ? pattern.test(entry.url) : entry.url === pattern)
FetchMock.callCount = () => callLog.length
server/workers/ScrapeQueue.js
+475
View File
@@ -0,0 +1,475 @@
//
// The scrape worker.
//
// Owns the whole lifecycle for one URL: cache check -> robots -> polite fetch ->
// parse -> thumbnail -> persist -> notify. Saving a note enqueues URLs and
// returns immediately; nothing in the request path waits on the network.
//
// Structural fixes over the old implementation:
// - Promise.allSettled semantics instead of a completion counter, so a single
// rejection can no longer strand a batch forever.
// - Every fetch goes through HostQueue, so there is no unbounded fan-out.
// - Jobs for the same URL coalesce into one fetch, fanned out to every
// attachment row that wanted it.
// - Pending work is recoverable after a restart from attachment.scrape_state.
//
const config = require('@config/scrape')
const db = require('@config/database')
const UrlSafety = require('@helpers/UrlSafety')
const HttpFetch = require('@helpers/HttpFetch')
const HostQueue = require('@helpers/HostQueue')
const Robots = require('@helpers/Robots')
const Realtime = require('@helpers/Realtime')
const Schema = require('@helpers/Schema')
const SiteScrape = require('@helpers/SiteScrape')
const ScrapeCache = require('@models/ScrapeCache')
const cheerio = require('cheerio')
let ScrapeQueue = module.exports = {}
// attachment.scrape_state
const STATE = ScrapeQueue.STATE = { PENDING: 0, FETCHING: 1, DONE: 2, FAILED: 3 }
let settings = { ...config }
let queue = HostQueue.shared
let started = false
// urlKey -> { promise, jobs: [] } so concurrent requests for one URL share a fetch.
const inFlight = new Map()
// userId -> pending job count, for backpressure.
const perUser = new Map()
// userId -> debounce timer, so a 10-link note triggers one refresh not ten.
const emitTimers = new Map()
//
// Every job promise currently outstanding. Needed because callers like
// recoverPending() and scanTextForWebsites() deliberately do not await their
// batches - without this, drain() could return while fire-and-forget work was
// still sitting in front of its first await.
//
const outstanding = new Set()
const EMIT_DEBOUNCE_MS = 1500
const now = () => Math.round(Date.now() / 1000)
//
// Test seam. Lets a test run with zero politeness spacing and no robots lookups
// while still exercising the real code path.
//
ScrapeQueue.configure = (overrides = {}) => {
settings = { ...config, ...overrides }
queue = new HostQueue.Queue({
globalConcurrency: settings.maxConcurrentFetches,
perHostConcurrency: settings.perHostConcurrency,
minSpacingMs: settings.perHostMinSpacingMs,
maxSpacingMs: settings.perHostMaxSpacingMs,
})
return settings
}
ScrapeQueue.__state = () => ({
inFlight: inFlight.size,
perUser: new Map(perUser),
queue: queue.stats(),
started,
})
ScrapeQueue.__reset = () => {
inFlight.clear()
outstanding.clear()
perUser.clear()
emitTimers.forEach(timer => clearTimeout(timer))
emitTimers.clear()
settings = { ...config }
queue = HostQueue.shared
started = false
}
ScrapeQueue.start = () => {
started = true
return ScrapeQueue
}
ScrapeQueue.stop = () => {
started = false
queue.stop()
emitTimers.forEach(timer => clearTimeout(timer))
emitTimers.clear()
}
//
// Resolves once every queued and in-flight job has settled.
//
// Loops because jobs enqueue follow-up work: a page fetch spawns a thumbnail
// fetch, and a recovery batch is registered asynchronously.
//
ScrapeQueue.drain = async () => {
for (let pass = 0; pass < 20; pass++) {
await Promise.allSettled([...outstanding])
await queue.drain()
await Promise.allSettled([...inFlight.values()].map(entry => entry.promise))
if (!outstanding.size && !inFlight.size && !queue.size() && !queue.inflight()) {
return
}
}
}
ScrapeQueue.size = () => queue.size() + inFlight.size + outstanding.size
//
// Debounced notification. The client already refetches on these events, so one
// per user per window is enough - the old code emitted once per URL.
//
const notifyUser = (userId) => {
if (emitTimers.has(userId)) { return }
const timer = setTimeout(() => {
emitTimers.delete(userId)
Realtime.emitToUser(userId, 'update_counts')
Realtime.emitToUser(userId, 'update_note_attachments')
}, EMIT_DEBOUNCE_MS)
if (timer.unref) { timer.unref() }
emitTimers.set(userId, timer)
}
// Tests want the emit without waiting out the debounce.
ScrapeQueue.__flushNotifications = () => {
for (const [userId, timer] of [...emitTimers.entries()]) {
clearTimeout(timer)
emitTimers.delete(userId)
Realtime.emitToUser(userId, 'update_counts')
Realtime.emitToUser(userId, 'update_note_attachments')
}
}
const setAttachmentState = async (attachmentId, state) => {
if (!attachmentId) { return }
await db.promise().query(
'UPDATE attachment SET scrape_state = ?, scrape_attempts = scrape_attempts + 1 WHERE id = ?',
[state, attachmentId]
)
}
//
// Write a successful result onto an attachment row.
//
// attachment.text stays PLAINTEXT - the FULLTEXT index depends on it.
//
const applyResultToAttachment = async (attachmentId, fields) => {
if (!attachmentId) { return }
await db.promise().query(
`UPDATE attachment
SET text = ?, url = ?, last_indexed = ?, file_location = ?, scrape_state = ?,
scrape_attempts = scrape_attempts + 1
WHERE id = ?`,
[fields.text, fields.url, now(), fields.fileLocation, STATE.DONE, attachmentId]
)
}
//
// A failed scrape must not leave a sentinel string in the search index. The old
// code stored 'No site text' / 'Request Timeout' in attachment.text, which then
// got concatenated into the note's searchable content.
//
const applyFailureToAttachment = async (attachmentId, error) => {
if (!attachmentId) { return }
const kind = (error && error.kind) || 'unknown'
// A blocked or permanently-gone URL has no preview to show, ever. Drop the
// shell row rather than leaving an orphan showing the bare URL.
const terminal = kind === 'blocked' || kind === 'robots_disallowed' ||
(kind === 'http' && (error.status === 404 || error.status === 410))
if (terminal) {
await db.promise().query('DELETE FROM attachment WHERE id = ? AND scrape_state IN (?, ?)',
[attachmentId, STATE.PENDING, STATE.FETCHING])
return
}
await setAttachmentState(attachmentId, STATE.FAILED)
}
//
// Fetch and store the preview image. Never fatal: a page with no usable
// thumbnail is still a perfectly good link preview.
//
const fetchThumbnail = async (imageUrl, pageUrl) => {
if (!imageUrl) { return null }
// Lazy require: Attachment requires this module, so a top-level require
// would be a cycle.
const Attachment = require('@models/Attachment')
try {
return await queue.schedule(hostKeyFor(imageUrl), () =>
Attachment.downloadThumbnail(imageUrl, pageUrl))
} catch (error) {
console.log('[scrape] thumbnail failed', imageUrl, error.kind || error.message)
return null
}
}
const hostKeyFor = (url) => {
try { return UrlSafety.registrableHost(new URL(url).hostname) }
catch (error) { return '(none)' }
}
//
// Fetch one URL and write it to the cache. Returns the cache row fields.
//
const performFetch = async (url) => {
const cached = await ScrapeCache.get(url)
// Fresh and usable - zero network I/O.
if (ScrapeCache.isFresh(cached) && ScrapeCache.isUsable(cached)) {
return { fromCache: true, fields: ScrapeCache.toAttachmentFields(cached) }
}
// Parked failure still inside its backoff window.
if (ScrapeCache.isFresh(cached) && cached.fetch_state !== 'ok') {
const error = new Error(cached.last_error || 'previously failed')
error.kind = cached.fetch_state === 'blocked' ? 'blocked' : 'http'
error.status = cached.status || null
throw error
}
if (settings.checkRobots !== false) {
const allowed = await Robots.isAllowed(url)
if (!allowed) {
const error = new Error(`robots.txt disallows ${url}`)
error.kind = 'robots_disallowed'
await ScrapeCache.recordFailure(url, error)
throw error
}
}
let spacingMs = null
if (settings.checkRobots !== false) {
const crawlDelay = await Robots.crawlDelayMs(url)
if (Robots.exceedsCrawlDelayCap(crawlDelay)) {
const error = new Error(`Crawl-delay of ${crawlDelay}ms exceeds cap`)
error.kind = 'blocked'
await ScrapeCache.recordFailure(url, error)
throw error
}
spacingMs = crawlDelay
}
await ScrapeCache.markPending(url)
const conditional = ScrapeCache.conditionalFor(cached)
let response = null
try {
response = await queue.schedule(hostKeyFor(url), () => HttpFetch.fetchDocument(url, {
timeoutMs: settings.pageTimeoutMs,
budgetMs: settings.totalBudgetMs,
maxBytes: settings.maxHtmlBytes,
conditional,
}), { spacingMs })
} catch (error) {
await ScrapeCache.recordFailure(url, error)
throw error
}
// Unchanged since last time - reuse what we already stored.
if (response.notModified && cached) {
await ScrapeCache.touchNotModified(url)
const refreshed = await ScrapeCache.get(url)
return { fromCache: true, notModified: true, fields: ScrapeCache.toAttachmentFields(refreshed) }
}
const $ = cheerio.load(response.html || '')
const parsed = SiteScrape.parse($, response.finalUrl || url)
// Reuse an existing thumbnail file rather than refetching the image.
let fileLocation = (cached && cached.image_url === parsed.imageUrl && cached.file_location)
? cached.file_location
: await fetchThumbnail(parsed.imageUrl, response.finalUrl || url)
await ScrapeCache.putSuccess(url, {
...parsed,
status: response.status,
etag: response.etag,
lastModified: response.lastModified,
contentType: response.contentType,
fileLocation,
})
return {
fromCache: false,
fields: {
text: parsed.searchText,
fileLocation,
// Keep the URL the user wrote. The canonical URL is recorded in the
// cache row; rewriting attachment.url would change link behaviour.
url,
},
}
}
//
// Coalesce: one network fetch per URL no matter how many notes asked for it.
//
const fetchCoalesced = (url) => {
const key = UrlSafety.urlKey(url)
if (!key) { return Promise.reject(Object.assign(new Error(`unusable url ${url}`), { kind: 'blocked' })) }
const existing = inFlight.get(key)
if (existing) { return existing.promise }
const promise = performFetch(url).finally(() => inFlight.delete(key))
inFlight.set(key, { promise, url })
return promise
}
const releaseUser = (userId) => {
const count = (perUser.get(userId) || 0) - 1
if (count <= 0) { perUser.delete(userId) } else { perUser.set(userId, count) }
}
//
// Queue one URL. Resolves with the search text (or '' on failure) and never
// rejects, so a caller can await a whole batch without special handling.
//
ScrapeQueue.enqueueUrl = ({ userId, noteId, attachmentId, url }) => {
const job = runJob({ userId, noteId, attachmentId, url })
// Registered synchronously, so drain() sees the job even before it reaches
// its first await.
outstanding.add(job)
job.finally(() => outstanding.delete(job))
return job
}
const runJob = async ({ userId, noteId, attachmentId, url }) => {
if (!settings.enabled) { return '' }
// Backstop, so no future caller can reach the DB layer unguarded when the
// scrape columns are absent.
if (!Schema.isScrapeSchemaReady()) { return '' }
const pending = perUser.get(userId) || 0
if (pending >= settings.maxPendingPerUser) {
// Left at scrape_state = 0, so recoverPending() picks it up later.
// Deferred, not dropped.
console.log(`[scrape] deferring ${url} - user ${userId} at pending cap`)
return ''
}
perUser.set(userId, pending + 1)
try {
await setAttachmentState(attachmentId, STATE.FETCHING)
const result = await fetchCoalesced(url)
await applyResultToAttachment(attachmentId, result.fields)
notifyUser(userId)
return result.fields.text || ''
} catch (error) {
await applyFailureToAttachment(attachmentId, error)
notifyUser(userId)
console.log(`[scrape] ${url} failed: ${error.kind || 'error'} ${error.message}`)
return ''
} finally {
releaseUser(userId)
}
}
//
// Queue a batch. allSettled, so one bad URL cannot strand the rest - this is
// the structural replacement for the old completion counter.
//
ScrapeQueue.enqueueUrls = async (jobs) => {
if (!jobs || !jobs.length) { return [] }
const results = await Promise.allSettled(jobs.map(job => ScrapeQueue.enqueueUrl(job)))
return results.map(result => result.status === 'fulfilled' ? result.value : '')
}
//
// Wait for one URL already queued, with a hard cap. Used by the bookmarklet so
// it keeps responding after the scrape without being able to hang forever.
//
ScrapeQueue.awaitUrl = async (promise, capMs) => {
let timer = null
const timeout = new Promise(resolve => {
timer = setTimeout(() => resolve('__scrape_timeout__'), capMs || settings.totalBudgetMs)
if (timer.unref) { timer.unref() }
})
try {
return await Promise.race([promise, timeout])
} finally {
if (timer) { clearTimeout(timer) }
}
}
//
// Crash recovery.
//
// The queue is in memory, so a pm2 reload loses it - but every enqueued URL
// already has an attachment row, which makes the work re-derivable with no
// extra table. Rows stuck mid-fetch are reset, rows still pending are
// re-enqueued, and anything older than a day is retired so nothing
// recirculates forever.
//
ScrapeQueue.recoverPending = async () => {
if (!settings.enabled) { return { requeued: 0, retired: 0 } }
// Every query below references scrape_state, which only exists once an
// admin has run ./migrateDatabase.sh.
if (!Schema.isScrapeSchemaReady()) { return { requeued: 0, retired: 0 } }
// Anything left at FETCHING died with the previous process.
await db.promise().query(
'UPDATE attachment SET scrape_state = ? WHERE scrape_state = ?',
[STATE.PENDING, STATE.FETCHING]
)
const cutoff = now() - (24 * 60 * 60)
const [retiredResult] = await db.promise().query(
`UPDATE attachment SET scrape_state = ?
WHERE scrape_state = ? AND enqueued_at IS NOT NULL AND enqueued_at < ?`,
[STATE.FAILED, STATE.PENDING, cutoff]
)
const [rows] = await db.promise().query(
`SELECT id, user_id, note_id, url FROM attachment
WHERE attachment_type = 1 AND visible = 1 AND scrape_state = ?
ORDER BY enqueued_at IS NULL, enqueued_at
LIMIT ?`,
[STATE.PENDING, settings.maxQueueLength]
)
if (rows.length) {
console.log(`[scrape] recovering ${rows.length} pending URL(s) after restart`)
}
// Fire and forget: boot must not wait on the network.
// Fire and forget, but the individual jobs register themselves in
// `outstanding`, so drain() can still wait for them.
ScrapeQueue.enqueueUrls(rows.map(row => ({
userId: row.user_id,
noteId: row.note_id,
attachmentId: row.id,
url: row.url,
}))).catch(error => console.log('[scrape] recovery failed', error.message))
return { requeued: rows.length, retired: retiredResult.affectedRows || 0 }
}
shell.nix
+99 -5
View File
@@ -11,6 +11,17 @@ pkgs.mkShell {
pm2
mariadb
openssl
# Schema migrations and DB scaffolding only -- the running server never
# imports Prisma. The nixpkgs wrapper is self-configuring: it exports
# PRISMA_SCHEMA_ENGINE_BINARY pointing at a nix-built prisma-engines and
# execs the bundled CLI, so there is no engine download and none of the
# usual NixOS prebuilt-binary trouble.
#
# Pinned to prisma_7 deliberately, NOT the `prisma` alias: Prisma 8 has no
# MariaDB support yet, so if nixpkgs moves that alias to v8 this project's
# database would stop being supported underneath us.
prisma_7
];
shellHook = ''
@@ -21,6 +32,36 @@ pkgs.mkShell {
MYSQL_SOCKET="$PWD/.mysql/mysql.sock"
MYSQL_PIDFILE="$PWD/.mysql/mysql.pid"
#
# Work out what database situation we are in BEFORE touching anything.
# This shell is used on the production server too, where a system MariaDB
# is already running -- we must not initialise a datadir, start a second
# server on port 3306, or inject dev credentials over the real ones.
#
# ours - the project-local server is already up (another nix-shell)
# existing - some other server is listening; not ours to manage
# start - nothing is running; this is a dev box, bring one up
#
SS_DB_MODE="start"
if mysqladmin --socket="$MYSQL_SOCKET" ping &>/dev/null 2>&1; then
SS_DB_MODE="ours"
elif (echo >/dev/tcp/127.0.0.1/3306) &>/dev/null \
|| [ -S /run/mysqld/mysqld.sock ] \
|| [ -S /var/run/mysqld/mysqld.sock ] \
|| [ -S /tmp/mysql.sock ]; then
SS_DB_MODE="existing"
fi
if [ "$SS_DB_MODE" = "existing" ]; then
echo "Existing database detected -- not starting the project one."
echo "Leaving DB_* and JSON_KEY alone so the server's own config wins."
fi
STARTED_MYSQL=0
if [ "$SS_DB_MODE" = "start" ]; then
mkdir -p "$MYSQL_DATADIR"
# Initialize data directory on first run
@@ -34,9 +75,6 @@ pkgs.mkShell {
echo "Initialization complete."
fi
# Start mysqld if not already running
STARTED_MYSQL=0
if ! mysqladmin --socket="$MYSQL_SOCKET" ping &>/dev/null 2>&1; then
STARTED_MYSQL=1
echo "Starting MySQL..."
mysqld --no-defaults \
@@ -52,15 +90,56 @@ pkgs.mkShell {
done
echo "MySQL ready."
#
# Collation is explicit because the server default here is
# utf8mb4_unicode_ci while every application table is
# utf8mb4_general_ci. Without this, a table created later without an
# explicit COLLATE would silently inherit the wrong one, changing string
# comparison and FULLTEXT ranking. Prisma does not model collation and
# `prisma migrate diff` does not report the difference.
#
# Existing databases need a one-time, non-destructive fix:
# ALTER DATABASE application
# CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
#
mysql -u root --socket="$MYSQL_SOCKET" \
-e "CREATE DATABASE IF NOT EXISTS application;" 2>/dev/null
-e "CREATE DATABASE IF NOT EXISTS application \
CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" 2>/dev/null
fi
# Set app DB env vars — dotenv won't override these since they're already in process.env
#
# Dev credentials, injected ONLY when this shell is providing the database
# and there is no ~/.env supplying real config.
#
# This matters because dotenv does NOT override variables already present in
# process.env (server/config/database.js loads ~/.env). So anything exported
# here silently beats the server's real configuration -- including JSON_KEY,
# which would invalidate every existing user session.
#
if [ "$SS_DB_MODE" != "existing" ] && [ ! -f "$HOME/.env" ]; then
export DB_HOST=127.0.0.1
export DB_USER=root
export DB_PASS=
#
# Prisma CLI connection string, for migrations and scaffolding.
#
# The warning above about shadowing ~/.env does NOT apply here:
# DATABASE_URL is read only by the Prisma CLI, never by
# server/config/database.js, so it cannot affect the running server.
#
# Deliberately NOT derived from DB_USER/DB_PASS. Prisma needs DDL
# rights, and in production those variables hold the least-privilege
# `app` account. Admin credentials are composed by migrateDatabase.sh
# instead, which is why this export lives only in the dev branch.
#
export DATABASE_URL="mysql://root@127.0.0.1:3306/application"
# Stop the Prisma CLI phoning checkpoint.prisma.io for update checks.
# Keeps the schema tests network-free and the output uncluttered.
export CHECKPOINT_DISABLE=1
# Stable dev JWT key, persisted across shell sessions
if [ -z "''${JSON_KEY:-}" ]; then
if [ -s "$PWD/.mysql/jwt_key" ]; then
@@ -72,6 +151,10 @@ pkgs.mkShell {
fi
fi
elif [ -f "$HOME/.env" ]; then
echo "~/.env found -- using its DB_* and JSON_KEY, not dev defaults."
fi
# Restore a SQL dump — handles both plain .sql and .gz files
restore-db() {
local file="''${1:-}"
@@ -79,6 +162,16 @@ pkgs.mkShell {
echo "Usage: restore-db <backup.sql[.gz]>"
return 1
fi
# Refuse to run against a database this shell does not manage. The dumps
# are --all-databases: restoring one would overwrite every schema on the
# server and reset root's password. That is fine for a local dev copy and
# catastrophic on the production box.
if [ "$SS_DB_MODE" = "existing" ]; then
echo "Refusing: this shell is talking to a pre-existing database." >&2
echo "restore-db only runs against the project-local dev database." >&2
return 1
fi
echo "Restoring database from $file ..."
if [[ "$file" == *.gz ]]; then
gunzip -c "$file" | mysql -u root --socket="$MYSQL_SOCKET"
@@ -99,6 +192,7 @@ pkgs.mkShell {
export -f restore-db
# Auto-restore if --argstr backupFile was passed to nix-shell
# (restore-db itself refuses when the database is not ours)
if [ -n "${backupFile}" ]; then
restore-db "${backupFile}"
fi