Testing Migrations

applyLastBackup.sh

@@ -0,0 +1,62 @@
+echo '-------'
+echo 'Starting Database Restore'
+echo '-------'
+
+#get Latest database backup
+
+# Unzip File
+# gzip -dk file.gz
+
+source ~/.env
+
+echo "$PROD_DB_PASS" 
+
+BACKUPDIR="/home/mab/databaseBackupSolidScribe"
+
+mkdir -p $BACKUPDIR
+cd $BACKUPDIR
+
+# -t sort by modification time, newest first
+# -A --almost-all, do not list implied . and ..
+LASTZIPPEDFILE=$(ls -At *.gz | head -n1)
+
+# -k keep file after unzip
+# -d Decompress
+# -v verbose
+echo "Unzipping $LASTZIPPEDFILE"
+gunzip -dkv $LASTZIPPEDFILE
+
+BACKUPFILE=$(ls -At *.sql | head -n1)
+
+#Fix to replace incompatible DB type
+echo "Updating table name in -> $BACKUPFILE"
+#sed -i $BACKUPFILE -e 's/utf8mb4_0900_ai_ci/utf8mb4_unicode_ci/g'
+
+#Fix encoding for dev DB and exclude system tables
+# sed -i 's/utf8mb4_0900_ai_ci/utf8mb4_general_ci/g' $BACKUPFILE
+# sed -r '/INSERT INTO `(sys|mysql)`/d' $BACKUPFILE > $BACKUPFILE
+
+#echo "Removing and syncing static files"
+#rm -r /home/mab/ss/staticFiles/*
+#rsync -e 'ssh -p 13328' -hazC --update [email protected]:/home/mab/pi/staticFiles /home/mab/ss/
+
+echo "Updating Database"
+mysql -u root --password="$PROD_DB_PASS" < $BACKUPFILE
+
+## Optimize Database Tables
+# mysqlcheck --all-databases
+mysqlcheck --all-databases -o -u root --password="$PROD_DB_PASS" #--silent
+# mysqlcheck --all-databases --auto-repair
+# mysqlcheck --all-databases --analyze
+
+# Fix an issues with DB after messing around with it
+mysql_upgrade -u root --password="$PROD_DB_PASS"
+
+#clean up extracted and modified SQL dumps
+rm *.sql
+
+
+
+echo '-------'
+echo "Applied Prod database to Dev. LastFile: $BACKUPFILE"
+echo '-------'

applyProdDatabaseToDev.sh

@@ -1,5 +1,3 @@
-#!/bin/bash
-
 echo '-------'
 echo 'Starting Database Restore'
 echo '-------'
@@ -12,7 +10,7 @@ echo '-------'
 BACKUPDIR="/home/mab/databaseBackupSolidScribe"
 #DEVDBPASS="Crama!Lama*Jamma###88383!!!!!345345956245i"
 #DEVDBPASS="RootPass1234!"
-DEVDBPASS="ReallySecureRootPass123!"
+DEVDBPASS="LetsKeepOut69FatHorses!"
 # LazaLinga&33Can't!Do!That34
 
 cd $BACKUPDIR
@@ -39,14 +37,14 @@ sed -r '/INSERT INTO `(sys|mysql)`/d' $BACKUPFILE > $BACKUPFILE
 
 echo "Removing and syncing static files"
 rm -r /home/mab/ss/staticFiles/*
-rsync -e 'ssh -p 13328' -hazC --update mab@solidscribe.com:/home/mab/pi/staticFiles /home/mab/ss/
+rsync -e 'ssh -p 13328' -hazC --update mab@45.79.41.75:/home/mab/pi/staticFiles /home/mab/ss/
 
 echo "Updating Database"
 mysql -u root --password="$DEVDBPASS" < $BACKUPFILE
 
 ## Optimize Database Tables
 # mysqlcheck --all-databases
-mysqlcheck --all-databases -o -u root --password="$DEVDBPASS" --silent
+mysqlcheck --all-databases -o -u root --password="$DEVDBPASS" #--silent
 # mysqlcheck --all-databases --auto-repair
 # mysqlcheck --all-databases --analyze
 

backupDatabase.sh

@@ -1,15 +1,15 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Take all variables in .env and turn them into local variables for this script
-source ~/.env
+source /home/mab/.env
 
-BACKUPDIR="/home/mab/databaseBackupSolidScribe"
+# BACKUPDIR defined in .env
 
 mkdir -p $BACKUPDIR
 cd $BACKUPDIR
 
 NOW=$(date +"%Y-%m-%d_%H-%M")
-ssh [email protected] -p 13328 "mysqldump --all-databases --single-transaction --user root -p$PROD_DB_PASS" > "backup-$NOW.sql"
+mysqldump --all-databases --single-transaction --user root --password="$PROD_DB_PASS" > "backup-$NOW.sql"
 gzip "backup-$NOW.sql"
 
 # cp "backup-$NOW.sql" "/mnt/Windows Data/DatabaseBackups/backup-$NOW.sql"
@@ -17,7 +17,7 @@ gzip "backup-$NOW.sql"
 echo "Database Backup Complete on $NOW"
 
 # Delete all but last 8 files
-ls -tp | grep -v '/$' | tail -n +9 | tr '\n' '\0' | xargs -0 rm --
+ls -tp | grep -v '/$' | tail -n +9 | tr '\n' '\0' | xargs -0 rm -f --
 
 ## 
 # Restore DB

buildAndUpdateProd.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-
-# 
-# Push built release files to production server
-#
-
-echo -e "\e[32m\nStarting Build. \n\e[0m"
-
-# Build out new release
-cd client
-npm run build
-cd ..
-
-# only compress client/dist and server with node_modules
-echo -e "\e[32m\nCompressing client and server code... \n\e[0m"
-tar -czf release.tar.gz server node_modules client/dist staticFiles/assets
-
-#send compressed release to remote machine
-echo -e "\e[32m\nMoving compressed release to production... \n\e[0m"
-rsync -e 'ssh -p 13328' -havzC --update release.tar.gz [email protected]:/home/mab/pi/
-
-# Remove Release from local after its been uploaded
-rm release.tar.gz
-
-#uncompress release on server
-echo -e "\e[32m\nExtracting release on production... \n\e[0m"
-ssh [email protected] -p 13328 "cd /home/mab/pi/; rm -r server node_modules client; tar -xzf *.tar.gz --overwrite; rm *.tar.gz; pm2 reload all"
-
-#Congratulate how awesome you are
-echo -e "\e[32m\nRelease Complete! Nice Work! \n\e[0m"

client/.gitignore

@@ -1,26 +1,28 @@
-.DS_Store
+# Logs
-node_modules
+logs
-/dist
+*.log
-
-
-# local env files
-.env.local
-.env.*.local
-*.pem
-*.crt
-*.key
-
-# Log files
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
 
 # Editor directories and files
+.vscode/*
+!.vscode/extensions.json
 .idea
-.vscode
+.DS_Store
 *.suo
 *.ntvs*
 *.njsproj
 *.sln
 *.sw?
+
+# Local dev TLS certs (mkcert)
+*.pem
+certs/

client/.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+  "recommendations": ["Vue.volar"]
+}

client/README.md

@@ -1,19 +1,5 @@
-# client
+# Vue 3 + Vite
 
-## Project setup
+This template should help get you started developing with Vue 3 in Vite. The template uses Vue 3 `<script setup>` SFCs, check out the [script setup docs](https://v3.vuejs.org/api/sfc-script-setup.html#sfc-script-setup) to learn more.
-```
-npm install
-```
 
-### Compiles and hot-reloads for development
+Learn more about IDE Support for Vue in the [Vue Docs Scaling up Guide](https://vuejs.org/guide/scaling-up/tooling.html#ide-support).
-```
-npm run serve
-```
-
-### Compiles and minifies for production
-```
-npm run build
-```
-
-### Customize configuration
-See [Configuration Reference](https://cli.vuejs.org/config/).

client/babel.config.js

@@ -1,5 +0,0 @@
-module.exports = {
-  presets: [
-    '@vue/cli-plugin-babel/preset'
-  ]
-}

client/index.html

@@ -0,0 +1,31 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
+    <meta name="description" content="Encrypted notes and scratch pad" />
+
+    <!-- PWA. The manifest is a real static file: it used to be injected at
+         runtime as a base64 data: URI from App.vue, with the production origin
+         hardcoded, which broke icons and scope outside production. -->
+    <link rel="manifest" href="/manifest.json" />
+    <meta name="theme-color" content="#000000" />
+
+    <link rel="icon" href="/favicon.ico" sizes="any" />
+    <link rel="icon" type="image/svg+xml" href="/img/icons/logo.svg" />
+
+    <!-- iOS has no share-target support, but still installs to the home screen.
+         The apple-touch icon must be the opaque variant -- iOS composites
+         transparent PNGs onto black and the dark half of the mark vanishes. -->
+    <link rel="apple-touch-icon" href="/img/icons/apple-touch-icon-180.png" />
+    <meta name="apple-mobile-web-app-capable" content="yes" />
+    <meta name="apple-mobile-web-app-title" content="Scribe" />
+    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
+
+    <title>Solid Scribe</title>
+  </head>
+  <body>
+    <div id="app"></div>
+    <script type="module" src="/src/main.js"></script>
+  </body>
+</html>

client/jsconfig.json

@@ -1,19 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "es5",
-    "module": "esnext",
-    "baseUrl": "./",
-    "moduleResolution": "node",
-    "paths": {
-      "@/*": [
-        "src/*"
-      ]
-    },
-    "lib": [
-      "esnext",
-      "dom",
-      "dom.iterable",
-      "scripthost"
-    ]
-  }
-}

client/package.json

@@ -1,32 +1,32 @@
 {
-  "name": "solidscribe",
+  "name": "solidscribe-client",
-  "version": "0.1.0",
   "private": true,
+  "version": "0.0.0",
+  "type": "module",
   "scripts": {
-    "serve": "vue-cli-service serve",
+    "dev": "vite",
-    "build": "vue-cli-service build"
+    "build": "vite build",
+    "preview": "vite preview"
   },
   "dependencies": {
+    "@tailwindcss/vite": "^4.1.13",
     "axios": "^1.1.3",
-    "core-js": "^3.6.5",
+    "chart.js": "^4.6.1",
-    "es6-promise": "^4.2.8",
+    "diff-match-patch": "^1.0.5",
     "fomantic-ui-css": "^2.9.0",
-    "vue": "^2.6.11",
+    "mitt": "^3.0.1",
-    "vue-chartjs": "^5.0.1",
+    "socket.io-client": "^2.5.0",
-    "vue-router": "^3.2.0",
+    "tailwindcss": "^4.1.13",
-    "vuedraggable": "^2.24.3",
+    "vue": "^3.5.18",
-    "vuex": "^3.4.0"
+    "vue-chartjs": "^5.3.4",
+    "vue-router": "^4.0.0",
+    "vuedraggable": "^4.1.0",
+    "vuex": "^4.0.0"
   },
   "devDependencies": {
-    "@vue/cli-plugin-babel": "^5.0.8",
+    "@vitejs/plugin-vue": "^6.0.1",
-    "@vue/cli-plugin-router": "^5.0.8",
+    "vite": "^7.1.2",
-    "@vue/cli-plugin-vuex": "^5.0.8",
+    "vite-plugin-pwa": "^2.0.0",
-    "@vue/cli-service": "^5.0.8",
+    "workbox-window": "^7.4.1"
-    "vue-template-compiler": "^2.6.11"
+  }
-  },
-  "browserslist": [
-    "> 1%",
-    "last 2 versions",
-    "not dead"
-  ]
 }

client/pnpm-workspace.yaml

@@ -0,0 +1,2 @@
+allowBuilds:
+  esbuild: true

client/public/img/icons/logo-maskable.svg

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 132.29166 132.29167" width="512" height="512">
+  <rect x="0" y="0" width="132.29166" height="132.29167" fill="#ffffff"/>
+  <g transform="translate(13.229166,13.229167) scale(0.8)">
+    <g transform="translate(0,-164.70832)">
+    <path id="path3813-4" d="m 56.22733,165.36641 -55.56249926,15.875 8e-7,63.5 47.62499846,11.90625 v 27.78125 l -47.76066333,-13.9757 0.13566407,10.00695 55.56249926,15.875 v -47.625 l -47.6249985,-11.90625 -8e-7,-47.625 47.7606633,-13.94121 c 0.135664,-2.30629 -0.135664,-9.87129 -0.135664,-9.87129 z" style="fill:#0f7425;fill-opacity:1;stroke:none;stroke-width:0.5291667;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"/>
+    <path id="path4563" d="m 20.508581,220.92891 c 15.265814,-14.23899 27.809717,-7.68002 39.687499,3.96875 v -7.9375 C 51.75093,200.8366 37.512584,206.01499 20.508581,205.05391 Z" style="fill:#04cb03;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"/>
+    <path id="path4563-6" d="m 111.78985,220.92891 c -15.265834,-14.23899 -27.809737,-7.68002 -39.68752,3.96875 v -7.9375 c 8.445151,-16.12356 22.683497,-10.94517 39.68752,-11.90625 z" style="display:inline;fill:#04cb03;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"/>
+    <path id="path3813-4-2" d="m 76.07108,165.36641 55.5625,15.875 v 63.5 l -47.625,11.90625 v 27.78125 l 47.76067,-13.9757 -0.13567,10.00695 -55.5625,15.875 v -47.625 l 47.625,-11.90626 V 189.17891 L 75.93542,175.2377 c -0.13567,-2.30629 0.13566,-9.87129 0.13566,-9.87129 z" style="display:inline;fill:#04cb03;fill-opacity:1;stroke:none;stroke-width:0.52916676;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"/>
+    </g>
+  </g>
+</svg>

client/public/img/icons/logo.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 132.29166 132.29167" width="512" height="512">
+  <g transform="translate(0,-164.70832)">
+    <path id="path3813-4" d="m 56.22733,165.36641 -55.56249926,15.875 8e-7,63.5 47.62499846,11.90625 v 27.78125 l -47.76066333,-13.9757 0.13566407,10.00695 55.56249926,15.875 v -47.625 l -47.6249985,-11.90625 -8e-7,-47.625 47.7606633,-13.94121 c 0.135664,-2.30629 -0.135664,-9.87129 -0.135664,-9.87129 z" style="fill:#0f7425;fill-opacity:1;stroke:none;stroke-width:0.5291667;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"/>
+    <path id="path4563" d="m 20.508581,220.92891 c 15.265814,-14.23899 27.809717,-7.68002 39.687499,3.96875 v -7.9375 C 51.75093,200.8366 37.512584,206.01499 20.508581,205.05391 Z" style="fill:#04cb03;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"/>
+    <path id="path4563-6" d="m 111.78985,220.92891 c -15.265834,-14.23899 -27.809737,-7.68002 -39.68752,3.96875 v -7.9375 c 8.445151,-16.12356 22.683497,-10.94517 39.68752,-11.90625 z" style="display:inline;fill:#04cb03;fill-opacity:1;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"/>
+    <path id="path3813-4-2" d="m 76.07108,165.36641 55.5625,15.875 v 63.5 l -47.625,11.90625 v 27.78125 l 47.76067,-13.9757 -0.13567,10.00695 -55.5625,15.875 v -47.625 l 47.625,-11.90626 V 189.17891 L 75.93542,175.2377 c -0.13567,-2.30629 0.13566,-9.87129 0.13566,-9.87129 z" style="display:inline;fill:#04cb03;fill-opacity:1;stroke:none;stroke-width:0.52916676;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"/>
+  </g>
+</svg>

client/public/index.html

@@ -1,69 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width,initial-scale=1.0">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-
-    <link rel="icon" href="/api/static/assets/favicon.ico" type="image/ico"/>
-    <link rel="shortcut icon" href="/api/static/assets/favicon.ico" type="image/x-icon"/>
-
-    <meta name="theme-color" content="#000" />
-    <link rel="manifest" href="/api/static/assets/manifest.json">
-
-    <title>Solid Scribe - An easy, encrypted Note App</title>
-    <!-- <title><%= htmlWebpackPlugin.options.title %></title> -->
-  </head>
-  <body>
-    <noscript>
-      <strong>We're sorry but Solid Scribe doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
-    </noscript>
-    <div id="app">
-      <!-- placeholder data for scrapers with no JS -->
-      <style>
-        body {
-          background-color: #212221;
-          color: #aeaeae;
-          height: 100vh;
-          width: 100%;
-        }
-        .centered {
-          position: fixed;
-          top: 50%;
-          left: 50%;
-          transform: translate(-50%, -50%);
-          text-align: center;
-          font-family: Arial, Helvetica, sans-serif;
-        }
-        .logo {
-          width: 200px;
-          height: auto;
-        }
-        .scrape-info {
-          opacity: 0;
-        }
-      </style>
-    
-      <div class="centered">
-        <img class="logo" src="/api/static/assets/logo.svg" alt="Solid Scribe Logo - if you can read this your connection is really slow">
-        <h1>Solid Scribe</h1>
-        <h3>An easy, encrypted Note App</h3>
-        <h4>Loading...</h4>
-      </div>
-
-      <div class="scrape-info">
-        <h1>Solid Scribe</h1>
-        <h2>A note application that respects your privacy.</h2>
-        <p>Take notes with a clean editor that works on desktop or mobile.</p>
-        <p>Search notes, links and files to find what you need.</p>
-        <p>Accessable everywhere.</p>
-        <p>Categorize notes with tags.</p>
-        <p>Share data with fellow users.</p>
-        <p>Encrypt notes for additional security.</p>
-        <b>This site requires Javascipt to run.</b>
-      </div>
-
-    </div>
-    <!-- built files will be auto injected -->
-  </body>
-</html>

client/public/manifest.json

@@ -0,0 +1,78 @@
+{
+  "id": "/",
+  "name": "Solid Scribe",
+  "short_name": "Scribe",
+  "description": "Encrypted notes and scratch pad",
+  "start_url": "/",
+  "scope": "/",
+  "display": "standalone",
+  "display_override": [
+    "standalone",
+    "minimal-ui"
+  ],
+  "orientation": "any",
+  "theme_color": "#000000",
+  "background_color": "#000000",
+  "icons": [
+    {
+      "src": "/img/icons/icon-192.png",
+      "sizes": "192x192",
+      "type": "image/png",
+      "purpose": "any"
+    },
+    {
+      "src": "/img/icons/icon-512.png",
+      "sizes": "512x512",
+      "type": "image/png",
+      "purpose": "any"
+    },
+    {
+      "src": "/img/icons/maskable-192.png",
+      "sizes": "192x192",
+      "type": "image/png",
+      "purpose": "maskable"
+    },
+    {
+      "src": "/img/icons/maskable-512.png",
+      "sizes": "512x512",
+      "type": "image/png",
+      "purpose": "maskable"
+    }
+  ],
+  "shortcuts": [
+    {
+      "name": "Scratch Pad",
+      "short_name": "Scratch",
+      "url": "/quick",
+      "icons": [
+        {
+          "src": "/img/icons/icon-192.png",
+          "sizes": "192x192",
+          "type": "image/png"
+        }
+      ]
+    },
+    {
+      "name": "Files & Links",
+      "short_name": "Files",
+      "url": "/attachments",
+      "icons": [
+        {
+          "src": "/img/icons/icon-192.png",
+          "sizes": "192x192",
+          "type": "image/png"
+        }
+      ]
+    }
+  ],
+  "share_target": {
+    "action": "/share-target",
+    "method": "GET",
+    "enctype": "application/x-www-form-urlencoded",
+    "params": {
+      "title": "title",
+      "text": "text",
+      "url": "url"
+    }
+  }
+}

client/public/robots.txt

@@ -1,2 +0,0 @@
-User-agent: *
-Disallow:

client/src/App.vue

@@ -1,7 +1,7 @@
 <template>
 	<div id="app" :class="{ 'night-mode':($store.getters.getIsNightMode == 2) }">
 
-		<div class="ui container" v-if="showFakeSite">
+		<!-- <div class="ui container" v-if="showFakeSite">
 			<div class="ui basic very padded segment">
 			<div class="ui inverted red segment">
 				<h1>WARNING - False site detected</h1>
@@ -11,7 +11,7 @@
 				<h2>You will be redirected to the correct domain in {{redirectSeconds}}</h2>
 			</div>
 			</div>
-		</div>
+		</div> -->
 
 		<div class="auth-block" v-if="requireAuth">
 			<div class="ui raised inverted segment">
@@ -51,19 +51,24 @@
 
 		<global-notification />
 
+		<update-prompt />
+
 	</div>
 </template>
 
 <script>
 
-// import io from 'socket.io-client'
 import axios from 'axios'
+import GlobalSiteMenu from '@/components/GlobalSiteMenu.vue'
+import GlobalNotification from '@/components/GlobalNotificationComponent.vue'
+import UpdatePrompt from '@/components/UpdatePrompt.vue'
 
 export default {
 	name: 'App',
 	components: {
-		'global-site-menu': require('@/components/GlobalSiteMenu.vue').default,
+		'global-site-menu': GlobalSiteMenu,
-		'global-notification':require('@/components/GlobalNotificationComponent.vue').default,
+		'global-notification': GlobalNotification,
+		'update-prompt': UpdatePrompt,
 	},
 	data: function(){ 
 		return {
@@ -172,16 +177,18 @@ export default {
 	},
 	mounted: function(){
 
-		const isDev = process.env['NODE_ENV'] == 'development'
+		// Vite does not shim node's `process` into the browser bundle the way
-		if(window.location.hostname.toLowerCase().replace('www.','') != "solidscribe.com" && !isDev){
+		// webpack/vue-cli did; import.meta.env.DEV is the equivalent.
-			this.showFakeSite = true
+		const isDev = import.meta.env.DEV
-			setInterval(() => {
+		// if(window.location.hostname.toLowerCase().replace('www.','') != "solidscribe.com" && !isDev){
-				this.redirectSeconds--
+		// 	this.showFakeSite = true
-				if(this.redirectSeconds == 0){
+		// 	setInterval(() => {
-					window.location.href = 'https://www.solidscribe.com'
+		// 		this.redirectSeconds--
-				}
+		// 		if(this.redirectSeconds == 0){
-			}, 1000)
+		// 			window.location.href = 'https://www.solidscribe.com'
-		}
+		// 		}
+		// 	}, 1000)
+		// }
 
 		//Update totals for entire app on event
 		this.$io.on('update_counts', () => {

client/src/EventBus.js

@@ -1,10 +1,12 @@
-import Vue from 'vue'
+import mitt from 'mitt'
-const EventBus = new Vue()
 
-Object.defineProperties(Vue.prototype, {
+const emitter = mitt()
-	$bus: {
-		get: () => EventBus
-	}
-})
 
-export default EventBus;
+// Preserves the Vue 2 $on/$emit/$off API used throughout the codebase
+const EventBus = {
+	$on:   (...args) => emitter.on(...args),
+	$off:  (...args) => emitter.off(...args),
+	$emit: (...args) => emitter.emit(...args),
+}
+
+export default EventBus

client/src/Helpers.js

@@ -1,5 +1,3 @@
-import Vue from 'vue'
-
 const helpers = {}
 
 helpers.timeAgo = (time) => {
@@ -62,10 +60,9 @@ helpers.timeAgo = (time) => {
 }
 
 const plugin = {
-    install () {
+    install(app) {
-        Vue.helpers = helpers
+        app.config.globalProperties.$helpers = helpers
-        Vue.prototype.$helpers = helpers
     }
 }
 
-Vue.use(plugin)
+export default plugin

client/src/LaunchPage.js

@@ -0,0 +1,81 @@
+//
+// Launch page preference.
+//
+// The app used to inject its manifest at runtime with
+// `start_url: window.location.href`, so installing from /metrictrack made the
+// installed app always open there. That mattered: not everyone wants the notes
+// page as their home.
+//
+// A static manifest can't carry a per-install start_url, and we need a static
+// same-origin manifest for share_target to register at all (a data: URI
+// manifest has an opaque origin, so scope resolution fails). So the preference
+// moves in-app: the manifest's start_url is "/", and we redirect from there.
+//
+// This is also strictly better than the old behaviour -- the old one froze the
+// home page at install time and could only be changed by reinstalling. This one
+// is editable in Settings.
+//
+
+const LAUNCH_KEY = 'launchPage'
+const DEFAULT_LAUNCH = '/notes'
+
+// Pages that make sense as a home. Anything else (a specific note, the share
+// target, login) would be a bad place to land every time.
+export const LAUNCH_OPTIONS = [
+	{ path: '/notes', label: 'Notes' },
+	{ path: '/overview', label: 'Overview' },
+	{ path: '/quick', label: 'Scratch Pad' },
+	{ path: '/metrictrack', label: 'Metric Tracking' },
+	{ path: '/attachments', label: 'Files & Links' },
+]
+
+export function getLaunchPage(){
+	const stored = localStorage.getItem(LAUNCH_KEY)
+	const valid = LAUNCH_OPTIONS.some(option => option.path === stored)
+	return valid ? stored : DEFAULT_LAUNCH
+}
+
+export function setLaunchPage(path){
+	if(!LAUNCH_OPTIONS.some(option => option.path === path)){ return }
+	localStorage.setItem(LAUNCH_KEY, path)
+}
+
+export function isStandalone(){
+	return window.matchMedia('(display-mode: standalone)').matches
+		|| window.navigator.standalone === true
+}
+
+//
+// Capture the page the user installed from, matching the old behaviour: if they
+// install while on /metrictrack, that becomes their launch page. Only set as a
+// default -- an explicit choice in Settings is never overwritten.
+//
+export function watchForInstall(){
+	window.addEventListener('appinstalled', () => {
+		if(localStorage.getItem(LAUNCH_KEY)){ return }
+
+		const path = window.location.pathname
+		if(LAUNCH_OPTIONS.some(option => option.path === path)){
+			localStorage.setItem(LAUNCH_KEY, path)
+		}
+	})
+}
+
+//
+// "/" is a shim for installed launches. Redirect only when running standalone,
+// so the marketing home page still works normally in a browser tab.
+//
+export function installLaunchRedirect(router){
+
+	watchForInstall()
+
+	router.beforeEach((to, from, next) => {
+
+		// Only the very first navigation of a standalone launch
+		if(to.path === '/' && from.matched.length === 0 && isStandalone()){
+			return next(getLaunchPage())
+		}
+
+		return next()
+	})
+}

client/src/assets/semantic-helper.css

@@ -179,7 +179,7 @@ div.ui.basic.green.label {
 }
 /*Overwrites for modifiable theme color */
 i.green.icon.icon.icon.icon, i.green.icon.icon.icon.icon.icon {
-	color: var(--main-accent);
+	color: var(--main-accent) !important;
 }
 .button {
 	box-shadow: 2px 2px 4px -2px rgba(40, 40, 40, 0.89) !important;
@@ -1001,3 +1001,8 @@ padding-right: 10px;
 	background-color: rgba(0,0,0,0.7);
 	z-index: 1000;
 }
+
+
+i.outline {
+  outline-style: none;
+}

client/src/assets/vue.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="37.07" height="36" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 198"><path fill="#41B883" d="M204.8 0H256L128 220.8L0 0h97.92L128 51.2L157.44 0h47.36Z"></path><path fill="#41B883" d="m0 0l128 220.8L256 0h-51.2L128 132.48L50.56 0H0Z"></path><path fill="#35495E" d="M50.56 0L128 133.12L204.8 0h-47.36L128 51.2L97.92 0H50.56Z"></path></svg>

client/src/components/FileUploadButton.vue

@@ -18,12 +18,13 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 	import axios from 'axios'
 	export default {
 		name: 'FileUploadButton',
 		props: [ 'noteId' ],
 		components: {
-			'nm-button':require('@/components/NoteMenuButtonComponent.vue').default
+			'nm-button':defineAsyncComponent(() => import('@/components/NoteMenuButtonComponent.vue'))
 		},
 		data () {
 			return {

client/src/components/GlobalSiteMenu.vue

@@ -369,14 +369,15 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 
 	import axios from 'axios'
 
 	export default {
 		components: {
-			'search-input': require('@/components/SearchInput.vue').default,
+			'search-input': defineAsyncComponent(() => import('@/components/SearchInput.vue')),
-			'counter':require('@/components/AnimatedCounterComponent.vue').default,
+			'counter':defineAsyncComponent(() => import('@/components/AnimatedCounterComponent.vue')),
-			'logo':require('@/components/LogoComponent.vue').default,
+			'logo':defineAsyncComponent(() => import('@/components/LogoComponent.vue')),
 		},
 		data: function(){ 
 			return {
@@ -393,7 +394,7 @@
 		beforeMount(){
 			window.addEventListener('resize', this.resizeEventHandler)
 		},
-		beforeDestroy(){
+		beforeUnmount(){
 			window.removeEventListener('resize', this.resizeEventHandler)
 		},
 		mounted: function(){

client/src/components/LoginFormComponent.vue

@@ -153,8 +153,10 @@
 					this.$io.emit('user_connect', token)
 					localStorage.setItem('loginToken', token)
 
-					//Redirect user to notes section after login
+					//Redirect user to notes section after login, or back to
-					this.$router.push('/notes')
+					//wherever they were headed (e.g. an Android share waiting to
+					//be saved -- see ShareTargetPage.vue)
+					this.$router.push(this.$route.query.redirect || '/notes')
 				}
 			},
 			register(){

client/src/components/LogoComponent.vue

@@ -12,18 +12,6 @@
 	   width="500">
 	  <defs
 	     id="defs2" />
-	  <metadata
-	     id="metadata5">
-	    <rdf:RDF>
-	      <cc:Work
-	         rdf:about="">
-	        <dc:format>image/svg+xml</dc:format>
-	        <dc:type
-	           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-	        <dc:title></dc:title>
-	      </cc:Work>
-	    </rdf:RDF>
-	  </metadata>
 	  <g
 	     style="display:inline"
 	     transform="translate(0,-164.70832)"

client/src/components/Metrictracking/LineGraph.vue

@@ -0,0 +1,212 @@
+<template>
+	<div class="line-graph-holder">
+		<Line v-if="hasData" :data="chartData" :options="chartOptions" />
+		<div v-else class="no-data">No data for the selected range.</div>
+	</div>
+</template>
+
+<script>
+	import { Line } from 'vue-chartjs'
+	import {
+		Chart as ChartJS,
+		CategoryScale,
+		LinearScale,
+		PointElement,
+		LineElement,
+		Tooltip,
+		Legend,
+	} from 'chart.js'
+
+	ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Tooltip, Legend)
+
+	// Categorical slots, validated for both of this app's surfaces (#fff / #000)
+	// with scripts/validate_palette.js. Assigned by a field's stable position,
+	// never by its position in a particular graph, so toggling fields on and off
+	// never repaints the ones that stay.
+	const SERIES = {
+		light: ['#2a78d6','#eb6834','#1baf7a','#eda100','#e87ba4','#008300','#4a3aa7','#e34948'],
+		dark:  ['#3987e5','#d95926','#199e70','#c98500','#d55181','#008300','#9085e9','#e66767'],
+	}
+
+	// A field can carry an explicit Fomantic colour name chosen by the user; that
+	// is their intent and it matches the field's icon elsewhere on the page, so it
+	// wins over the generated slot.
+	const FOMANTIC = {
+		red:'#DB2828', orange:'#F2711C', yellow:'#FBBD08', olive:'#B5CC18',
+		green:'#21BA45', teal:'#00B5AD', blue:'#2185D0', violet:'#6435C9',
+		purple:'#A333C8', pink:'#E03997', brown:'#A5673F', grey:'#767676',
+		black:'#1B1C1D',
+	}
+
+	export default {
+		name: 'MetricLineGraph',
+		inject: ['getFieldColor'],
+		components: { Line },
+		props: {
+			graph:         { type: Object, required: true },  // { fieldIds: [...] }
+			userFields:    { type: Object, default: () => ({}) },
+			cycleData:     { type: Object, default: () => ({}) },
+			tempChartDays: { type: [Number, String], default: 30 },
+		},
+		computed: {
+			isDark(){
+				return Number(this.$store.getters.getIsNightMode) === 1
+			},
+			surface(){
+				return this.isDark ? '#000000' : '#ffffff'
+			},
+			textColor(){
+				// Text wears text tokens, never the series colour.
+				return this.isDark ? '#d9d9d9' : '#3d3d3d'
+			},
+			gridColor(){
+				// One step off the surface, hairline and recessive.
+				return this.isDark ? 'rgba(255,255,255,0.12)' : 'rgba(0,0,0,0.10)'
+			},
+			fieldIds(){
+				return this.graph?.fieldIds || []
+			},
+			// Every date present in the data, oldest first, trimmed to the
+			// requested window. Sorting by real date rather than object key
+			// order means out-of-order writes cannot scramble the line.
+			dateCodes(){
+				const codes = Object.keys(this.cycleData)
+					.map(code => ({ code, time: this.codeToDate(code) }))
+					.filter(entry => !isNaN(entry.time))
+					.sort((a, b) => a.time - b.time)
+					.map(entry => entry.code)
+
+				const span = parseInt(this.tempChartDays)
+				if(!span || span < 1 || span >= codes.length){ return codes }
+				return codes.slice(-span)
+			},
+			hasData(){
+				return this.fieldIds.length > 0 && this.dateCodes.length > 0
+			},
+			chartData(){
+				return {
+					labels: this.dateCodes.map(code => this.formatLabel(code)),
+					datasets: this.fieldIds.map(fieldId => {
+						const colour = this.colourFor(fieldId)
+						return {
+							label: this.userFields[fieldId]?.label || fieldId,
+							data: this.dateCodes.map(code => this.valueAt(code, fieldId)),
+							borderColor: colour,
+							backgroundColor: colour,
+							borderWidth: 2,
+							tension: 0.25,
+							// A marker that is drawn is >= 8px with a 2px ring in
+							// the surface colour; dense series only show them on
+							// hover so the line stays readable.
+							pointRadius: this.denseData ? 0 : 4,
+							pointHoverRadius: 5,
+							pointBorderColor: this.surface,
+							pointBorderWidth: 2,
+							pointHoverBorderColor: this.surface,
+							pointHoverBorderWidth: 2,
+							pointHitRadius: 12,
+							spanGaps: false, // a missing day is a gap, not a zero
+						}
+					}),
+				}
+			},
+			denseData(){
+				return this.dateCodes.length > 40
+			},
+			chartOptions(){
+				return {
+					responsive: true,
+					maintainAspectRatio: false,
+					animation: { duration: 200 },
+					interaction: { mode: 'index', intersect: false },
+					scales: {
+						x: {
+							grid: { display: false },
+							ticks: { color: this.textColor, maxRotation: 0, autoSkipPadding: 16 },
+							border: { color: this.gridColor },
+						},
+						y: {
+							grid: { color: this.gridColor, lineWidth: 1, drawTicks: false },
+							ticks: { color: this.textColor, padding: 8 },
+							border: { display: false },
+						},
+					},
+					plugins: {
+						// Always present: identity is never carried by colour alone,
+						// and it is the relief for the low-contrast light slots.
+						legend: {
+							display: true,
+							position: 'bottom',
+							labels: {
+								color: this.textColor,
+								usePointStyle: true,
+								pointStyle: 'circle',
+								boxWidth: 8,
+								boxHeight: 8,
+								padding: 14,
+							},
+						},
+						tooltip: {
+							backgroundColor: this.isDark ? '#1b1b1b' : '#ffffff',
+							titleColor: this.textColor,
+							bodyColor: this.textColor,
+							borderColor: this.gridColor,
+							borderWidth: 1,
+							padding: 10,
+							usePointStyle: true,
+							callbacks: {
+								label: ctx => {
+									const v = ctx.parsed.y
+									return `${ctx.dataset.label}: ${v === null ? '—' : v}`
+								},
+							},
+						},
+					},
+				}
+			},
+		},
+		methods: {
+			// Date codes are stored as D.M.YYYY
+			codeToDate(code){
+				const [d, m, y] = String(code).split('.').map(Number)
+				return new Date(y, (m || 1) - 1, d || 1).getTime()
+			},
+			formatLabel(code){
+				const date = new Date(this.codeToDate(code))
+				if(isNaN(date.getTime())){ return code }
+				return date.toLocaleDateString(undefined, { month: 'short', day: 'numeric' })
+			},
+			valueAt(code, fieldId){
+				const raw = this.cycleData[code]?.[fieldId]
+				if(raw === undefined || raw === null || raw === ''){ return null }
+				const num = Number(raw)
+				// Non-numeric field types cannot be plotted on a value axis; a gap
+				// is honest, where the old code substituted -1 and bent the scale.
+				return isNaN(num) ? null : num
+			},
+			colourFor(fieldId){
+				const named = this.getFieldColor ? this.getFieldColor(fieldId) : null
+				if(named && FOMANTIC[named]){ return FOMANTIC[named] }
+
+				// Stable slot: keyed off the field's position in userFields, so the
+				// colour follows the field rather than its rank in this graph.
+				const all = Object.keys(this.userFields)
+				const index = all.indexOf(fieldId)
+				const slots = this.isDark ? SERIES.dark : SERIES.light
+				return slots[(index < 0 ? 0 : index) % slots.length]
+			},
+		},
+	}
+</script>
+
+<style scoped>
+	.line-graph-holder {
+		width: 100%;
+		height: 320px;
+	}
+	.no-data {
+		padding: 2em 0;
+		text-align: center;
+		color: var(--menu-text, #767676);
+	}
+</style>

client/src/components/Metrictracking/MetricGraphsComponent.vue

@@ -43,12 +43,12 @@
 				</div>
 			</div>
 			<div class="sixteen wide column">
-				<dix class="ui basic segment" v-if="!editGraphs">
+				<div class="ui basic segment" v-if="!editGraphs">
 					<div class="ui button" v-on:click="toggleEditGraphs">
 						<i class="edit icon"></i>
 						<span>Add/Edit Graphs</span>
 					</div>
-				</dix>
+				</div>
 				
 
 				<div v-if="editGraphs">
@@ -64,7 +64,12 @@
 			</div>
 		</div>
 		
-		<div v-for="(graph, index) in graphs" :class="`ui not-padded ${editGraphs?'active ':'inactive '}segment animate-height`">
+		<div class="ui basic center aligned segment" v-if="!graphs || graphs.length == 0">
+			<h4 class="ui header">No graphs yet</h4>
+			<p>Choose <b>Add/Edit Graphs</b>, then <b>New Graph</b>, and pick which metrics to plot.</p>
+		</div>
+
+		<div v-for="(graph, index) in graphs" :class="`ui not-padded ${editGraphs?'active ':'inactive '}segment`">
 
 			<!-- Edit options -->
 			<div class="ui small header" v-if="editGraphs">
@@ -126,15 +131,20 @@
 				<span v-if="!editGraphs">Click "Edit Graphs" then,</span>
 				Select Graph type and Metrics to display
 			</div>
-			<div v-if="graph?.type == undefined && graph.fieldIds && graph.fieldIds.length > 0">
+			<div v-if="isLineGraph(graph) && graph.fieldIds && graph.fieldIds.length > 0">
-				<div :id="`graphdiv${index}`" style="width: 100%; min-height: 320px;"></div>
+				<LineGraph
+					:graph="graph"
+					:userFields="userFields"
+					:cycleData="cycleData"
+					:tempChartDays="tempChartDays"
+				/>
 			</div>
 
 			<div class="ui segment" v-if="editGraphs">
 
 				<!-- change graph type -->
 				<div v-for="(graphType, graphId) in graphTypesDef" class="ui buttons">
-					<div class="ui tiny button" v-on:click="changeGraphType(index, graphId)" :class="{'green':(String(graphId) == String(graph?.type))}">
+					<div class="ui tiny button" v-on:click="changeGraphType(index, graphId)" :class="{'green':isGraphTypeActive(graphId, graph)}">
 						{{ graphType }}
 					</div>
 				</div>
@@ -146,7 +156,7 @@
 					<span v-else v-on:click="toggleGraphField(fieldId, index)">
 						<i class="square outline icon"></i>
 					</span>
-					<i :class="`${$parent.getFieldColor(fieldId)} ${$parent.getFieldIcon(fieldId)} icon`"></i>
+					<i :class="`${getFieldColor(fieldId)} ${getFieldIcon(fieldId)} icon`"></i>
 					<b>{{ userFields[fieldId]?.label }}</b>
 				</div>
 			</div>
@@ -160,12 +170,12 @@
 				</div>
 			</div>
 			<div class="sixteen wide column">
-				<dix class="ui basic segment" v-if="!editGraphs">
+				<div class="ui basic segment" v-if="!editGraphs">
 					<div class="ui button" v-on:click="toggleEditGraphs">
 						<i class="edit icon"></i>
 						<span>Add/Edit Graphs</span>
 					</div>
-				</dix>
+				</div>
 				
 
 				<div v-if="editGraphs">
@@ -189,11 +199,13 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 
 	const PILL_CALENDAR = 'pillCalendar'
 	const LAST_DONE = 'lastDone'
 
 	export default {
+		inject: ['getFieldColor', 'getFieldIcon'],
 		name: 'MetricTrackingGraphs',
 		props: [
 			'tempChartDays', 	// Number of days to display
@@ -205,7 +217,8 @@
 			'editGraphs'		// boolean for edit or not edit graphs
 		],
 		components: {
-			'PillCalendarGraph':require('@/components/Metrictracking/PillCalendarGraph.vue').default,
+			'PillCalendarGraph':defineAsyncComponent(() => import('@/components/Metrictracking/PillCalendarGraph.vue')),
+			'LineGraph':defineAsyncComponent(() => import('@/components/Metrictracking/LineGraph.vue')),
 		},
 		data: function(){
 			return {
@@ -221,33 +234,9 @@
 			// Constants
 			this.PILL_CALENDAR = PILL_CALENDAR
 			this.LAST_DONE = LAST_DONE
-
-			// Include JS libraries
-			let graphsScript = document.createElement('script')
-			graphsScript.setAttribute('src', '//cdnjs.cloudflare.com/ajax/libs/dygraph/2.1.0/dygraph.min.js')
-      		document.head.appendChild(graphsScript)
 		},
 		mounted(){
 			this.localGraphData = this.graphs
-
-			this.graphCurrentData()
-		},
-		updated(){
-			// update graphs here? Or watch graphs prop
-		},
-		watch: {
-			// whenever question changes, this function will run
-			userFields(newFields, oldFields) {
-				// console.log([newFields, oldFields])
-				if( JSON.stringify(oldFields) == "{}" ){
-					this.graphCurrentData()
-				}
-			},
-			tempChartDays(newDays, oldDays){
-				if( newDays != oldDays ){
-					this.graphCurrentData()
-				}
-			},
 		},
 		methods: {
 			saveGraphs(){
@@ -267,10 +256,35 @@
 				this.$emit('toggleEditGraphs')
 			},
 			changeGraphType(index, newType){
-				console.log(index + ' change to ' + newType)
+
-				this.localGraphData[index]['type'] = newType
+				// graphTypesDef is an object, so its keys are strings and the
+				// default line graph arrives here as the STRING 'undefined'.
+				// Saved graphs represent the line graph as an absent type, so
+				// remove the key rather than storing a string that no render
+				// test can match.
+				if(this.isLineGraphType(newType)){
+					delete this.localGraphData[index].type
+				}
+				else {
+					this.localGraphData[index].type = newType
+				}
+
 				this.saveGraphs()
 			},
+			// A line graph is the default: an absent type. Accept the string
+			// form too so graphs saved by the older code still render.
+			isLineGraphType(type){
+				return type === undefined || type === null || type === '' || type === 'undefined'
+			},
+			isLineGraph(graph){
+				return this.isLineGraphType(graph?.type)
+			},
+			isGraphTypeActive(graphId, graph){
+				if(this.isLineGraphType(graphId)){
+					return this.isLineGraph(graph)
+				}
+				return String(graphId) === String(graph?.type)
+			},
 			addGraph(){
 				this.localGraphData.push({})
 				this.saveGraphs()
@@ -329,103 +343,6 @@
 
 				return title
 			},
-			graphCurrentData(){
-
-				// try again if dygraphs isn't loaded
-				if( typeof(window.Dygraph) != 'function' ){
-					setTimeout(() => {
-						this.graphCurrentData()
-					}, 100)
-
-					return
-				}
-
-				const graphOptions = {
-					interactionModel: {},
-					// pointClickCallback: function(e, pt){
-					// 	console.log(e)
-					// 	console.log(pt)
-					// 	console.log(this.getValue(pt.idx, 0))
-					// }
-				}
-
-				// Excel date format YYYYMMDD
-				const convertToExcelDate = (dateCode) => {
-					return dateCode
-					.split('.')
-					.reverse()
-					.map(item => String(item).padStart(2,0))
-					.join('')
-				}
-
-				// Generate set of keys for graph length
-				let dataKeys = Object.keys(this.cycleData)
-				dataKeys = dataKeys.splice(0, this.tempChartDays)
-				console.log(dataKeys)
-
-
-				// build CSV data for each graph
-				this.graphs.forEach((graph,index) => {
-
-					// only chart line graphs with dygraphs
-					if( graph.type != undefined ){
-						return
-					}
-					if( !graph.fieldIds ){
-						return
-					}
-
-					// CSV or path to a CSV file.
-					let dataString = ""
-
-					// Lookup graph field titles
-					let graphLabels = ['Date']
-					graph.fieldIds.forEach(fieldId => {
-						const graphLabel = this.userFields[fieldId]?.label
-						const escapedLabel = graphLabel.replaceAll(',','')
-						graphLabels.push(escapedLabel)
-					})
-					dataString += graphLabels.join(',') + '\n'
-
-					
-					// build each row, for each day
-					for (var i = 0; i < dataKeys.length; i++) {
-
-						let nextFragment = []
-						// push date code to first column
-						nextFragment.push(convertToExcelDate(dataKeys[i]))
-
-						graph.fieldIds.forEach(fieldId => {
-
-							const currentEntry = this.cycleData[dataKeys[i]]
-							let currentValue = currentEntry[fieldId]
-
-							// setup correct float graphing
-							if(fieldId == 'BT'){
-								// parse temp to fixed length float 00.00
-								currentValue = parseFloat(currentValue).toFixed(2)
-							}
-
-							if( currentValue == undefined ){
-								currentValue = -1
-							}
-
-							nextFragment.push(currentValue)
-								
-						})
-
-						dataString += nextFragment.join(',') + "\n"
-					}
-
-					
-					let graphDiv = document.getElementById("graphdiv"+index)
-					const g = new Dygraph(graphDiv, dataString ,graphOptions)
-
-				})
-
-				return
-				
-			},
 		}
 	}
 </script>

client/src/components/Metrictracking/PillCalendarGraph.vue

@@ -81,6 +81,13 @@
 
 	}
 
+	.days-since {
+		padding: 0.75em 0.5em 0.25em 0.5em;
+		font-size: 0.95em;
+	}
+	.days-since-row {
+		padding: 0.15em 0;
+	}
 	.pill-container {
 		width: 100%;
 	}
@@ -229,36 +236,47 @@
 
 				
 				
-				<span class="pill-container" v-for="(entry, dateCode) in getChartData" v-if="dateCode == `${day}.${calendar.month}.${calendar.year}`">
+				<template v-for="(entry, dateCode) in getChartData" :key="dateCode">
+				<span class="pill-container" v-if="dateCode == `${day}.${calendar.month}.${calendar.year}`">
+					<template v-for="(dayData, fieldId) in entry" :key="fieldId">
 					<span 
-						v-for="(dayData, fieldId) in entry" 
 						v-if="showZeroValuesCheck(dayData.value, fieldId)"
 						class="pill" 
-						:class="[$parent.$parent.getFieldColor(fieldId), { 
+						:class="[getFieldColor(fieldId), { 
 							'did-next':dayData.didNext, 
 							'did-last':dayData.didLast,
 							'last-high':dayData.lastHigh,
 							'next-high':dayData.nextHigh,
 							}]">
 						<!-- 'zero-day':dayData.value == lowestGraphValue, -->
-						<!-- <i v-if="dayData.value != 0" :class="`tiny ${$parent.$parent.getFieldColor(fieldId)} ${$parent.$parent.getFieldIcon(fieldId)} icon`"></i> -->
+						<!-- <i v-if="dayData.value != 0" :class="`tiny ${getFieldColor(fieldId)} ${getFieldIcon(fieldId)} icon`"></i> -->
 						<!-- <span v-else class="icon-spacer"></span> 
 							:style="{height:(Math.round(dayData.value*5)+'px')}"
 
 						-->
 						<span v-if="dayData.value > lowestGraphValue-1" class="big-day">
-							<i v-if="!hideIcons" :class="`tiny white ${$parent.$parent.getFieldIcon(fieldId)} icon`"></i>
+							<i v-if="!hideIcons" :class="`tiny white ${getFieldIcon(fieldId)} icon`"></i>
 							<span v-if="!hideValues">
 								{{ getDayValue(fieldId, dayData.value) }}
 							</span>
 						</span>
 					</span>
+					</template>
 				</span>
+				</template>
 				<!-- <span v-for="fieldId in graph.fieldIds"></span> -->
 
 			</div>
 		</div>
 
+		<!-- time since each metric was last recorded -->
+		<div class="days-since" v-if="daysSince.length > 0">
+			<div class="days-since-row" v-for="entry in daysSince" :key="entry.fieldId">
+				<i :class="`${entry.color} ${entry.icon} icon`"></i>
+				<span v-if="entry.days === null">Days Since {{ entry.label }} - no entries yet</span>
+				<span v-else>Days Since {{ entry.label }} - {{ entry.days }} {{ entry.days === 1 ? 'day' : 'days' }}.</span>
+			</div>
+		</div>
 
 	</div>
 </template>
@@ -269,6 +287,7 @@
 	// let chartData = {}
 
 	export default {
+		inject: ['getFieldColor', 'getFieldIcon', 'getFieldLabel'],
 		props: [
 			'graph', // options associated with this graph
 			'userFields', // all field attributes
@@ -309,6 +328,53 @@
 			this.setupCalendar(new Date())
 		},
 		computed: {
+			// "Days Since <metric> - <n> days" for every field on this graph.
+			// Counts a day as recorded only if it passes the same
+			// showZeroValuesCheck the pills use, so the "Show/Hide Lowest Value"
+			// toggle moves these numbers too.
+			daysSince(){
+
+				// showZeroValuesCheck reads lowestGraphValue, which getChartData
+				// assigns as a side effect -- touch it so the threshold is current
+				// and so this recomputes whenever the chart data does.
+				void this.getChartData
+
+				const dayMs = 86400000
+				const midnight = (time) => {
+					const date = new Date(time)
+					date.setHours(0, 0, 0, 0)
+					return date.getTime()
+				}
+				const today = midnight(Date.now())
+
+				return (this.graph?.fieldIds || []).map(fieldId => {
+
+					let latest = null
+
+					Object.keys(this.cycleData || {}).forEach(dateCode => {
+
+						const value = this.cycleData[dateCode]?.[fieldId]
+						if(value === undefined || value === null || value === ''){ return }
+
+						// honour the Lowest Value toggle
+						if(!this.showZeroValuesCheck(value, fieldId)){ return }
+
+						const time = this.dateCodeToTime(dateCode)
+						// ignore unparseable codes and anything dated ahead of today
+						if(time === null || time > today){ return }
+
+						if(latest === null || time > latest){ latest = time }
+					})
+
+					return {
+						fieldId,
+						label: this.getFieldLabel(fieldId),
+						icon: this.getFieldIcon(fieldId),
+						color: this.getFieldColor(fieldId),
+						days: latest === null ? null : Math.round((today - latest) / dayMs),
+					}
+				})
+			},
 			getChartData(){
 
 				let chartData = {}
@@ -389,11 +455,26 @@
 			},
 		},
 		methods: {
+			// Date codes are stored unpadded as D.M.YYYY
+			dateCodeToTime(dateCode){
+
+				const parts = String(dateCode).split('.').map(Number)
+				if(parts.length !== 3 || parts.some(isNaN)){ return null }
+
+				const [day, month, year] = parts
+				const date = new Date(year, month - 1, day)
+				date.setHours(0, 0, 0, 0)
+
+				return isNaN(date.getTime()) ? null : date.getTime()
+			},
 			showZeroValuesCheck(dayValue, fieldId){
 
+				// Optional chaining: a graph can reference a default field (BT,
+				// CM, ...) that was never customised, so it has no userFields
+				// entry and the bare lookup threw.
 				// if graph type is boolean or there are two options
-				let isBooleanField = this.userFields[fieldId].type == 'boolean'
+				let isBooleanField = this.userFields[fieldId]?.type == 'boolean'
-				if(this.userFields[fieldId].customOptions){
+				if(this.userFields[fieldId]?.customOptions){
 					let options = this.userFields[fieldId].customOptions
 					
 					isBooleanField = options.split(',').length == 2
@@ -411,6 +492,13 @@
 				}
 
 
+				// lowestGraphValue comes from Math.min of the charted values, so
+				// it is Infinity when the window holds no data -- without the
+				// finite guard nothing would ever clear the threshold.
+				if(!isFinite(this.lowestGraphValue)){
+					return true
+				}
+
 				return this.showZeroValues || dayValue > this.lowestGraphValue
 			},
 			getDayValue(fieldId, value){

client/src/components/NoteInputPanel.vue

@@ -62,7 +62,7 @@
 		<div class="menu-bottom-half" :class="{ 'hide-text':(openNotes > 1) }">
 
 			<!--
-			<div class="edit-button" v-on:click="$router.push(`/notes/open/${noteid}/menu/table`)" data-tooltip="Insert Table" data-position="bottom center">
+			<div class="edit-button" v-on:click="openMenuRoute('table')" data-tooltip="Insert Table" data-position="bottom center">
 				<i class="border all icon"></i>
 			</div> -->
 
@@ -87,13 +87,13 @@
 			<div class="edit-divide"></div>
 
 			<!-- 				
-			<div class="edit-button" v-on:click="$router.push(`/notes/open/${noteid}/menu/colors`)" data-tooltip="Note Color" data-position="top center" :style="{ 'background-color':styleObject['noteBackground'], 'color':styleObject['noteText']}">
+			<div class="edit-button" v-on:click="openMenuRoute('colors')" data-tooltip="Note Color" data-position="top center" :style="{ 'background-color':styleObject['noteBackground'], 'color':styleObject['noteText']}">
 				<i class="paint brush icon"></i>
 			</div> -->
-			<!-- <div class="edit-button" v-on:click="$router.push(`/notes/open/${noteid}/menu/tags`)" data-tooltip="Tags" data-position="top center">
+			<!-- <div class="edit-button" v-on:click="openMenuRoute('tags')" data-tooltip="Tags" data-position="top center">
 				<i class="tags icon"></i>
 			</div> -->
-			<div class="edit-button" v-on:click="$router.push(`/notes/open/${noteid}/menu/images`)" data-tooltip="Images" data-position="top center">
+			<div class="edit-button" v-on:click="openMenuRoute('images')" data-tooltip="Images" data-position="top center">
 				<i class="image icon"></i>
 				<span>Images</span>
 			</div>
@@ -105,7 +105,7 @@
 
 			<div class="edit-divide"></div>
 
-			<div class="edit-button" v-on:click="$router.push(`/notes/open/${noteid}/menu/options`)" data-tooltip="More Options" data-position="top center">
+			<div class="edit-button" v-on:click="openMenuRoute('options')" data-tooltip="More Options" data-position="top center">
 				&nbsp;&nbsp;
 				<i class="ellipsis horizontal icon"></i>
 				&nbsp;&nbsp;
@@ -157,7 +157,7 @@
 
 				<!-- tags on the side, only show on desktop -->
 				<div class="note-mini-tag-area"
-					v-on:click="$router.push(`/notes/open/${noteid}/menu/tags`)" 
+					v-on:click="openMenuRoute('tags')" 
 					:style="{ 'background-color':styleObject['noteBackground'], 'color':styleObject['noteText'] }">
 
 					<span class="add-mini-tag" v-if="allTags.length == 0">
@@ -201,7 +201,7 @@
 
 				<!-- Squire Box -->
 				<div 
-					id="squire-id" 
+					:id="`squire-id-${noteid}`" 
 					class="squire-box" 
 					ref="squirebox" 
 					:style="{ 'background-color':styleObject['noteBackground'], 'color':styleObject['noteText'] }"
@@ -359,33 +359,32 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 
-	const Squire = require('../assets/squire.js')
+	import '../assets/squire.js' // side-effect: sets window.Squire
-
+	const Squire = window.Squire
 	import axios from 'axios'
-	// const crypto = require('crypto')
+	import DiffMatchPatch from 'diff-match-patch'
-	const DiffMatchPatch = require('../../../server/helpers/DiffMatchPatch')
+	const dmp = new DiffMatchPatch()
-	const dmp = new DiffMatchPatch.diff_match_patch()
 
 	import SquireButtonFunctions from '@/mixins/SquireButtonFunctions.js'
 
-	let rawNoteText = '' // Used for comparing and generating diffs
 	
 	export default {
 	name: 'NoteInputPanel',
 		props: [ 'noteid', 'position', 'openMenu', 'urlData', 'openNotes'],
 		components:{
-			'note-tag-edit': () => import('@/components/NoteTagEdit.vue'),
+			'note-tag-edit': defineAsyncComponent(() => import('@/components/NoteTagEdit.vue')),
-			'color-picker': () => import('@/components/ColorPicker.vue'),
+			'color-picker': defineAsyncComponent(() => import('@/components/ColorPicker.vue')),
-			'file-upload-button': () => import('@/components/FileUploadButton.vue'),
+			'file-upload-button': defineAsyncComponent(() => import('@/components/FileUploadButton.vue')),
-			// 'delete-button': () => import('@/components/NoteDeleteButtonComponent.vue'),
+			// 'delete-button': defineAsyncComponent(() => import('@/components/NoteDeleteButtonComponent.vue')),
-			'side-slide-menu': () => import('@/components/SideSlideMenuComponent.vue'),
+			'side-slide-menu': defineAsyncComponent(() => import('@/components/SideSlideMenuComponent.vue')),
-			'simple-attachment-note': () => import('@/components/SimpleAttachmentNoteComponent.vue'),
+			'simple-attachment-note': defineAsyncComponent(() => import('@/components/SimpleAttachmentNoteComponent.vue')),
-			'share-note-component': () => import('@/components/ShareNoteComponent.vue'),
+			'share-note-component': defineAsyncComponent(() => import('@/components/ShareNoteComponent.vue')),
 
-			'color-tooltip':require('@/components/TextColorTooltipComponent.vue').default,
+			'color-tooltip':defineAsyncComponent(() => import('@/components/TextColorTooltipComponent.vue')),
-			'nm-button':require('@/components/NoteMenuButtonComponent.vue').default,
+			'nm-button':defineAsyncComponent(() => import('@/components/NoteMenuButtonComponent.vue')),
-			'loading-icon':require('@/components/LoadingIconComponent.vue').default,
+			'loading-icon':defineAsyncComponent(() => import('@/components/LoadingIconComponent.vue')),
 		},
 		mixins:[ SquireButtonFunctions ],
 		data(){
@@ -419,6 +418,9 @@
 
                 //All the squire settings
                 editor: null,
+                // Baseline for diff generation. Per-instance: as a module-scoped
+                // variable this was shared by every open note's panel.
+                rawNoteText: '',
                 usersOnNote: 0,
 
                 sideMenuOpen: false,
@@ -475,25 +477,17 @@
 		},
 		beforeMount(){
 
-			this.$bus.$on('new_file_upload', ({noteId, imageCode}) => {
+			// Named handlers so beforeUnmount can remove only THIS panel's.
-				if(this.noteid == noteId && this.editor){
+			// mitt's off(type) with no handler removes every panel's.
-					this.editor.moveCursorToEnd()
+			this.$bus.$on('new_file_upload', this.onFileUpload)
-					this.editor.insertHTML(imageCode)
+			this.$bus.$on('close_note_by_id', this.onCloseNoteById)
-					this.save()
-				}
-			})
-
-			this.$bus.$on('close_note_by_id', (noteId) => {
-				if(noteId == this.noteid){
-					this.closeButtonAction()
-				}
-			})
 		},
-		beforeDestroy(){
+		beforeUnmount(){
 
-			this.$io.emit('leave_room', this.rawTextId)
+			this.$io.emit('leave_room', { noteId: this.noteid })
 
-			this.$bus.$off('new_file_upload')
+			this.$bus.$off('new_file_upload', this.onFileUpload)
+			this.$bus.$off('close_note_by_id', this.onCloseNoteById)
 
 			this.destroyAttachmentStyles()
 
@@ -533,13 +527,13 @@
 		},
 		methods: {
 			testModify(){
-				const text = document.getElementById('squire-id').children[0].innerHTML
+				const text = this.$refs.squirebox.children[0].innerHTML
 				const prepended = 'Hello-> ' + text
-				document.getElementById('squire-id').children[0].innerHTML = prepended
+				this.$refs.squirebox.children[0].innerHTML = prepended
 
-				const text1 = document.getElementById('squire-id').children[2].innerHTML
+				const text1 = this.$refs.squirebox.children[2].innerHTML
 				const prepended1 = 'Hey-> ' + text1
-				document.getElementById('squire-id').children[2].innerHTML = prepended1
+				this.$refs.squirebox.children[2].innerHTML = prepended1
 			},
 			simulateTyping(index = 0){
 
@@ -593,7 +587,7 @@
 				this.setText(this.noteText)
 
 				// Use squire box HTML for diff/patch changes
-				rawNoteText = document.getElementById('squire-id').innerHTML
+				this.rawNoteText = this.$refs.squirebox.innerHTML
 
 				//focus on open, not on mobile, it causes the keyboard to pop up, thats annoying
 				if(!this.$store.getters.getIsUserOnMobile){
@@ -867,6 +861,8 @@
 			},
 			//Called on squire event for keyup
 			diffText(event){
+				console.log('Skipping diff text function')
+				return
 				// console.log(event.type)
 
 				const diffEvents = ['keyup','pathChange', 'click']
@@ -880,17 +876,17 @@
 				this.diffTextTimeout = setTimeout(() => {
 
 					// Current Editor Text
-					const liveEditorElm = document.getElementById('squire-id')
+					const liveEditorElm = this.$refs.squirebox
 
 					// virtual element for selecting div
 					let virtualEditorElm = document.createElement('div')
-					virtualEditorElm.innerHTML = rawNoteText
+					virtualEditorElm.innerHTML = this.rawNoteText
 
 					// element at cursor
 					const elmAtCaret = window.getSelection().getRangeAt(0).startContainer.parentNode
 
 					// Remove beginngin selector from path, make it more generic 
-					const path = this.generateSelector(elmAtCaret).replace('div#squire-id > ','')
+					const path = this.generateSelector(elmAtCaret).replace(`div#squire-id-${this.noteid} > `,'')
 					let workingPath = ''
 
 					// default to entire note text, select down if path
@@ -938,10 +934,10 @@
 					const patch_text = dmp.patch_toText(patch_list)
 
 					// save raw text for future diffs
-					rawNoteText = liveEditorElm.innerHTML
+					this.rawNoteText = liveEditorElm.innerHTML
 
 					let newPatch = {
-						id: this.rawTextId,
+						noteId: this.noteid,
 						diff: patch_text,
 						path: path,
 						// testing metrics
@@ -961,7 +957,10 @@
 				// console.log('incoming patches ', incomingPatchs)
 				return new Promise((resolve, reject) => {
 
-					const editorElement = document.getElementById('squire-id')
+					const editorElement = this.$refs.squirebox
+
+					// Socket-driven: this panel may not be rendered yet, or may be gone
+					if(!editorElement){ return resolve(false) }
 
 					// iterate over incoming patches because they apply to specific divs
 					incomingPatchs.forEach(patch => {
@@ -988,10 +987,10 @@
 					})
 
 					// save editor HTML after change for future comparisons
-					rawNoteText = editorElement.innerHTML
+					this.rawNoteText = editorElement.innerHTML
 
 					// update hash on patch
-					this.lastNoteHash = this.hashString( rawNoteText )
+					this.lastNoteHash = this.hashString( this.rawNoteText )
 
 					this.$nextTick(() => {
 						return resolve(true)
@@ -1041,7 +1040,7 @@
 
 					//tell websockets to truncate history at this save
 					this.lastNoteHash = currentHash //Update last saved note hash
-					this.$io.emit('truncate_diffs_at_save', {'rawTextId':this.rawTextId, 'hash':currentHash })
+					this.$io.emit('truncate_diffs_at_save', {'noteId':this.noteid, 'hash':currentHash })
 					
 					const postData = {
 						'noteId': this.currentNoteId,
@@ -1068,6 +1067,10 @@
 			},
 			loadNoteNextFromServer(){
 
+				if(this.statusText != 'saved'){
+					return
+				}
+
 				clearTimeout(this.reloadNoteDebounce)
 				this.reloadNoteDebounce = setTimeout(() => {
 
@@ -1075,13 +1078,28 @@
 					// this.setText('')
 
 					//Focus Regained on Note, check for update
+					console.log('Ajax Call to /api/note/get')
 					axios.post('/api/note/get', { 'noteId': this.noteid })
 					.then(response => {
 
+						// Skip note update if it didn't change.
+						const newNoteHash = this.hashString( response.data.text )
+						if(newNoteHash == this.lastNoteHash){
+							console.log('Skipping any changes. Notes match')
+							return
+						}
+
 						this.setupLoadedNoteData(response) 
 
-						//Manually set squire text to show
+						//Manually set squire text
-						this.setText(this.noteText)
+						const editorElement = this.$refs.squirebox
+						if(!editorElement){ return }
+						editorElement.innerHTML = response.data.text
+
+						this.$nextTick(() => {
+							this.restoreCaretPosition()
+							.catch(e => console.log(e) )
+						})
 						
 
 					})
@@ -1098,6 +1116,7 @@
 				//If user leaves page then returns to page, reload the first batch
 				if(checkForUpdateTimeout){
 
+					console.log('Focused then unfocused for greater than 3 seconds. Checking with server.')
 					this.loadNoteNextFromServer()
 					this.lastInteractionTimestamp = now
 				}
@@ -1127,6 +1146,15 @@
 
 				return hash;
 			},
+			// Menu routes are pushed (not replaced) because the history entry is
+			// load-bearing: every menu closes via $router.go(-1). Forwarding the
+			// current query keeps ?open= alive across the menu and the pop back.
+			openMenuRoute(menu){
+				this.$router.push({
+					path: `/notes/open/${this.noteid}/menu/${menu}`,
+					query: this.$route.query,
+				})
+			},
 			closeButtonAction(playAnimation = false){
 				this.sizeDown = playAnimation
 				const animationTimeout = (playAnimation ? 300 : 0)
@@ -1162,60 +1190,106 @@
 				})
 			},
 			destroyWebSockets(){
-				this.$io.removeListener('past_diffs')
+
-				this.$io.removeListener('update_user_count')
+				// By reference: removeListener(event) alone would remove the other
-				this.$io.removeListener('incoming_diff')
+				// open panels' handlers too. new_note_text_saved was never removed
-				this.$io.removeListener('update_note_attachments')
+				// at all and leaked a handler on every open/close.
+				this.$io.removeListener('update_user_count', this.onUpdateUserCount)
+				this.$io.removeListener('past_diffs', this.onPastDiffs)
+				this.$io.removeListener('incoming_diff', this.onIncomingDiff)
+				this.$io.removeListener('new_note_text_saved', this.onNewNoteTextSaved)
+				this.$io.removeListener('update_note_attachments', this.onUpdateNoteAttachments)
 
 				clearTimeout(this.idleNoteTimeout)
 			},
+			// Socket handlers are named methods rather than inline closures so that
+			// destroyWebSockets() can remove THIS panel's handler by reference.
+			// removeListener(event) with no handler drops every open panel's.
+			onFileUpload({noteId, imageCode}){
+				if(this.noteid == noteId && this.editor){
+					this.editor.moveCursorToEnd()
+					this.editor.insertHTML(imageCode)
+					this.save()
+				}
+			},
+			onCloseNoteById(noteId){
+				if(noteId == this.noteid){
+					this.closeButtonAction()
+				}
+			},
+			onUpdateUserCount(userCount){
+				this.usersOnNote = userCount
+			},
+			//Apply all diffs since last save
+			onPastDiffs(diffSinceLastUpdate){
+
+				// Every open panel shares one socket, so filter to this note's room
+				if(!this.isDiffForThisNote(diffSinceLastUpdate)){ return }
+
+				if(diffSinceLastUpdate != null){
+					this.diffsApplied = diffSinceLastUpdate.length
+				}
+
+				this.patchText(diffSinceLastUpdate)
+				.then(() => {
+					this.restoreCaretPosition()
+					.catch(e => console.log(e) )
+				})
+			},
+			onIncomingDiff(incomingDiff){
+
+				if(!this.isDiffForThisNote([incomingDiff])){ return }
+
+				this.patchText([incomingDiff])
+				.then(() => {
+					this.restoreCaretPosition()
+					.catch(e => console.log(e) )
+				})
+			},
+			// A diff payload carries the raw-text id it belongs to. Anything without
+			// one is dropped rather than guessed at -- applying a patch to the wrong
+			// note is how note bodies got clobbered.
+			isDiffForThisNote(diffs){
+
+				if(diffs == null){ return false }
+
+				const list = [].concat(diffs).filter(diff => diff != null)
+				if(list.length == 0){ return false }
+
+				return list.every(diff => String(diff.rawTextId) === String(this.rawTextId))
+			},
+			onNewNoteTextSaved({noteId, hash}){
+
+				if(this.statusText != 'saved'){
+					return
+				}
+
+				const sameIdCheck = (this.idleNote && this.noteid == noteId)
+				const differentHashCheck = (hash != this.lastNoteHash)
+
+				// if hashes do not match, reload text from server
+				if(sameIdCheck && differentHashCheck){
+					this.loadNoteNextFromServer()
+				}
+			},
+			onUpdateNoteAttachments(){
+				this.getAttachments()
+			},
 			initWebsocketEvents(){
 
 				//Tell server to push this note into a room
-				this.$io.emit('join_room', this.rawTextId )
+				// Ask for a note, not a room. The server authorises against the
-
+				// token and derives the room itself.
-				this.$io.on('update_user_count', userCount => {
+				this.$io.emit('join_room', {
-					this.usersOnNote = userCount
+					noteId: this.noteid,
-				})
+					token: localStorage.getItem('loginToken'),
-
-				//Apply all diffs since last save
-				this.$io.on('past_diffs', diffSinceLastUpdate => {
-
-					if(diffSinceLastUpdate != null){
-						this.diffsApplied = diffSinceLastUpdate.length
-						// console.log('Got Diffs Total -> ', diffSinceLastUpdate)
-					}
-					// console.log(diffSinceLastUpdate)
-					this.patchText(diffSinceLastUpdate)
-					.then(() => {
-						this.restoreCaretPosition()
-					})
-				})
-
-				this.$io.on('incoming_diff', incomingDiff => {
-
-					this.patchText([incomingDiff])
-					.then(() => {
-						this.restoreCaretPosition()
-					})
-				})
-
-				this.$io.on('new_note_text_saved', ({noteId, hash}) => {
-
-					const sameIdCheck = (this.idleNote && this.noteid == noteId)
-					const differentHashCheck = (hash != this.lastNoteHash)
-
-					// if hashes do not match, reload text from server
-					if(sameIdCheck && differentHashCheck){
-						this.loadNoteNextFromServer()
-					}
-
-				})
-
-				this.$io.on('update_note_attachments', () => {
-					this.getAttachments()
 				})
 
+				this.$io.on('update_user_count', this.onUpdateUserCount)
+				this.$io.on('past_diffs', this.onPastDiffs)
+				this.$io.on('incoming_diff', this.onIncomingDiff)
+				this.$io.on('new_note_text_saved', this.onNewNoteTextSaved)
+				this.$io.on('update_note_attachments', this.onUpdateNoteAttachments)
 			},
 			logNoteInteraction(){
 
@@ -1228,6 +1302,8 @@
 			},
 			saveCaretPosition(event){
 
+				return
+
 				//Find index of child element based on past range
 				const element = window.getSelection().getRangeAt(0).startContainer.parentNode
 
@@ -1235,6 +1311,8 @@
 				this.lastRange = this.generateSelector(element)
 				this.startOffset = parseInt(event.range.startOffset) || 0
 
+				console.log('Save Range ', [element, this.lastRange, this.startOffset])
+
 				return
 			},
 			restoreCaretPosition(){
@@ -1244,7 +1322,9 @@
 
 					if(!this.lastRange){ return resolve(true) }
 
-					const editorElement = document.getElementById('squire-id')
+					const editorElement = this.$refs.squirebox
+					if(!editorElement){ return resolve(true) }
+
 					const lastElement = editorElement.querySelector(this.lastRange)
 
 					if( !lastElement ){ return resolve(true) }

client/src/components/NoteTitleDisplayCard.vue

@@ -184,6 +184,7 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 
 	import axios from 'axios'
 
@@ -191,9 +192,9 @@
 	name: 'NoteTitleDisplayCard',
 		props: [ 'onClick', 'data', 'currentlyOpen', 'textResults', 'attachmentResults', 'tagResults', 'titleView' ],
 		components: {
-			'delete-button': require('@/components/NoteDeleteButtonComponent.vue').default,
+			'delete-button': defineAsyncComponent(() => import('@/components/NoteDeleteButtonComponent.vue')),
-			'note-tag-edit': require('@/components/NoteTagEdit.vue').default,
+			'note-tag-edit': defineAsyncComponent(() => import('@/components/NoteTagEdit.vue')),
-			'side-slide-menu': require('@/components/SideSlideMenuComponent.vue').default,
+			'side-slide-menu': defineAsyncComponent(() => import('@/components/SideSlideMenuComponent.vue')),
 		},
 		methods:{
 			cardClicked(){

client/src/components/SearchInput.vue

@@ -73,7 +73,6 @@
 					</div>
 				</div>
 			</div>
-		</div>
 
 	</span>
 </template>
@@ -104,7 +103,7 @@
 				this.tagSuggestions = []
 			})
 		},
-		beforeDestroy(){
+		beforeUnmount(){
 			this.$bus.$off('reset_fast_filters')
 		},
 		mounted: function(){

client/src/components/SideSlideMenuComponent.vue

@@ -107,11 +107,12 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 	export default {
 		name: 'SideSlideMenu',
 		props: [ 'name', 'styleObject', 'fullShadow', 'skipHistory' ],
 		components: {
-			'nm-button':require('@/components/NoteMenuButtonComponent.vue').default
+			'nm-button':defineAsyncComponent(() => import('@/components/NoteMenuButtonComponent.vue'))
 		},
 		data () {
 			return {
@@ -129,7 +130,7 @@
 				}
 			})
 		},
-		beforeDestroy(){
+		beforeUnmount(){
 		},
 		mounted(){
 

client/src/components/SvgDisplayer.vue

@@ -37,7 +37,7 @@
 </script>
 
 <style type="text/css" scoped>
-	.svg-holder >>> svg {
+	.svg-holder :deep(svg) {
 		width: 100%;
 		max-height: 250px;
 	}

client/src/components/TextColorTooltipComponent.vue

@@ -86,11 +86,12 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 
 
 	export default {
 		components:{
-			'nm-button':require('@/components/NoteMenuButtonComponent.vue').default
+			'nm-button':defineAsyncComponent(() => import('@/components/NoteMenuButtonComponent.vue'))
 		},
 		props: [ 'lastUsedColor' ],
 		data: function(){ 

client/src/components/UpdatePrompt.vue

@@ -0,0 +1,62 @@
+<template>
+	<div class="update-prompt" v-if="visible">
+		<span class="update-text">A new version is ready.</span>
+		<div class="ui tiny green button" v-on:click="apply()">Update</div>
+		<div class="ui tiny basic button" v-on:click="dismiss()">Later</div>
+	</div>
+</template>
+
+<script>
+	export default {
+		name: 'UpdatePrompt',
+		data(){
+			return { visible: false }
+		},
+		// The service worker is configured with skipWaiting:false, so a new build
+		// sits waiting until the user accepts it here rather than swapping assets
+		// out from under an open note mid-edit.
+		beforeMount(){
+			this.$bus.$on('sw-update-available', this.onUpdateAvailable)
+		},
+		beforeUnmount(){
+			this.$bus.$off('sw-update-available', this.onUpdateAvailable)
+		},
+		methods: {
+			onUpdateAvailable(){
+				this.visible = true
+			},
+			apply(){
+				this.visible = false
+				// Imported lazily: this component mounts with the app, but the
+				// worker registers after mount.
+				import('@/registerServiceWorker').then(({ updateSW }) => updateSW(true))
+			},
+			dismiss(){
+				// Stays waiting; the hourly update check will offer it again.
+				this.visible = false
+			},
+		},
+	}
+</script>
+
+<style scoped>
+	.update-prompt {
+		position: fixed;
+		left: 50%;
+		bottom: 1em;
+		transform: translateX(-50%);
+		z-index: 2000;
+		display: flex;
+		align-items: center;
+		gap: 0.75em;
+		padding: 0.75em 1em;
+		border: 1px solid var(--border_color);
+		border-radius: 6px;
+		background-color: var(--small_element_bg_color);
+		color: var(--text_color);
+		box-shadow: 0 2px 10px rgba(0,0,0,0.3);
+	}
+	.update-text {
+		white-space: nowrap;
+	}
+</style>

client/src/main.js

@@ -1,71 +1,51 @@
-// The Vue build version to load with the `import` command
+import { createApp } from 'vue'
-// (runtime-only or standalone) has been set in webpack.base.conf with an alias.
+import App from './App.vue'
-
-import Vue from 'vue'
-
-import Vuex from 'vuex'
-import store from './stores/mainStore';
-
-import App from './App'
 import router from './router'
+import store from './stores/mainStore'
 
-//Include entire fomantic ui library
-// import 'fomantic-ui-css/semantic.css';
-
-//Required site and reset CSS
 import 'fomantic-ui-css/components/reset.min.css'
-import 'fomantic-ui-css/components/site.css' //modified to remove included LATO fonts
+import 'fomantic-ui-css/components/site.css'
-
-//Only include parts that are used
 import 'fomantic-ui-css/components/button.min.css'
 import 'fomantic-ui-css/components/container.min.css'
 import 'fomantic-ui-css/components/form.min.css'
 import 'fomantic-ui-css/components/grid.min.css'
 import 'fomantic-ui-css/components/header.min.css'
-import 'fomantic-ui-css/components/icon.css' //Modified to remove brand icons
+import 'fomantic-ui-css/components/icon.css'
 import 'fomantic-ui-css/components/input.min.css'
 import 'fomantic-ui-css/components/segment.min.css'
 import 'fomantic-ui-css/components/label.min.css'
 
+import './assets/semantic-helper.css'
 
-//Overwrite and site styles and themes and good stuff
+// Last, so Tailwind utilities win specificity ties against Fomantic
-require('./assets/semantic-helper.css')
+import './tailwind.css'
-// Fonts 
-require('./assets/roboto-latin.woff2')
-require('./assets/roboto-latin-bold.woff2')
 
-
+import io from 'socket.io-client'
-
-
-//Import socket io, init using nginx configured socket path
-import io from 'socket.io-client';
-const socket = io({ path:'/socket' });
-
-//integrate connected socket into vue instance
-Object.defineProperties(Vue.prototype, {
-	$io: {
-		get: () => socket
-	}
-})
-
-
-
-// This callback runs before every route change, including on page load.
-// Sets the title of the page using vue router
-router.beforeEach((to, from, next) => {
-	document.title = to.meta.title + ' - Solid Scribe';
-	next();
-});
-
-//Attach event bus to main vue object, all components will inherit event bus
 import EventBus from './EventBus'
 import Helpers from './Helpers'
+import { installLaunchRedirect } from './LaunchPage'
 
-Vue.use(Vuex)
+const socket = io({ path: '/socket' })
-Vue.config.productionTip = false
 
-new Vue({
+router.beforeEach((to, from, next) => {
-	router,
+	document.title = to.meta.title + ' - Solid Scribe'
-	store,
+	next()
-	render: h => h(App),
+})
-}).$mount('#app')
+
+// Installed launches land on "/" and are redirected to the user's chosen
+// launch page -- see LaunchPage.js
+installLaunchRedirect(router)
+
+const app = createApp(App)
+
+app.config.globalProperties.$io = socket
+app.config.globalProperties.$bus = EventBus
+
+app.use(router)
+app.use(store)
+app.use(Helpers)
+
+app.mount('#app')
+
+// After mount, so a failed registration can never block the app rendering
+import('./registerServiceWorker')

client/src/mixins/SquireButtonFunctions.js

@@ -171,7 +171,7 @@ const SquireButtonFunctions = {
 			//
 
 			//Fetch the container
-			let container = document.getElementById('squire-id')
+			let container = this.$refs.squirebox
 			this.$router.go(-1)
 
 			setTimeout(()=>{
@@ -189,7 +189,7 @@ const SquireButtonFunctions = {
 			//
 
 			//Fetch the container
-			let container = document.getElementById('squire-id')
+			let container = this.$refs.squirebox
 
 			//Close menu if user is on mobile, then sort list	
 			this.$router.go(-1)
@@ -249,7 +249,7 @@ const SquireButtonFunctions = {
 			//
 
 			//Fetch the container
-			let container = document.getElementById('squire-id')
+			let container = this.$refs.squirebox
 
 			//Close menu if user is on mobile	
 			this.$router.go(-1)
@@ -317,7 +317,7 @@ const SquireButtonFunctions = {
 			//
 
 			//Fetch the container
-			let container = document.getElementById('squire-id')
+			let container = this.$refs.squirebox
 
 			//Close menu if user is on mobile, then sort list	
 			this.$router.go(-1)
@@ -377,13 +377,13 @@ const SquireButtonFunctions = {
 			// this.diffNoteText = this.editor._getHTML()
 			
 			//Make sure all list items have draggable property
-			let container = document.getElementById('squire-id')
+			// let container = this.$refs.squirebox
-			let listItems = container.getElementsByTagName('li')
+			// let listItems = container.getElementsByTagName('li')
-			for(let itemIndex in listItems){
+			// for(let itemIndex in listItems){
-				// console.log(listItems[itemIndex])
+			// 	// console.log(listItems[itemIndex])
-				// listItems[itemIndex].setAttribute('draggable','true')
+			// 	// listItems[itemIndex].setAttribute('draggable','true')
-			}
+			// }
-			// console.log(listItems)
+			// // console.log(listItems)
 
 		},
 		getText(){

client/src/pages/AttachmentsPage.vue

@@ -102,13 +102,14 @@
 	</div>
 </template>
 <script>
+import { defineAsyncComponent } from 'vue'
 
 	import axios from 'axios'
 
 	export default {
 		components: {
-			'attachment-display': require('@/components/AttachmentDisplayCard').default,
+			'attachment-display': defineAsyncComponent(() => import('@/components/AttachmentDisplayCard.vue')),
-			'counter':require('@/components/AnimatedCounterComponent.vue').default,
+			'counter':defineAsyncComponent(() => import('@/components/AnimatedCounterComponent.vue')),
 		},
 		data: function(){ 
 			return {
@@ -124,7 +125,7 @@
 			//
 			// Perform Login check
 			//
-			this.$parent.loginGateway()
+			this.$root.loginGateway()
 
 		},
 		mounted: function(){
@@ -140,7 +141,7 @@
 			//Mount notes on load if note ID is set
 			this.searchAttachments()
 		},
-		beforeDestroy(){
+		beforeUnmount(){
 
 			//Remove scroll event on destroy
 			window.removeEventListener('scroll', this.onScroll)

client/src/pages/BookmarkletPage.vue

@@ -40,13 +40,13 @@
 		},
 		beforeCreate: function(){
 			// Perform Login check
-			this.$parent.loginGateway()
+			this.$root.loginGateway()
 
 		},
 		mounted: function(){
 			this.getBookmarklet()
 		},
-		beforeDestroy(){
+		beforeUnmount(){
 
 		},
 		methods: {

client/src/pages/CycletrackingPage.vue

@@ -0,0 +1,298 @@
+<style>
+	.calendar {
+		width: 300px;
+		height: 400px;
+		display: inline-block;
+	}
+	.day {
+		width: 40px;
+		height: 40px;
+		border: 1px solid var(--border_color);
+		display: inline-block;
+		text-align: center;
+		font-size: 1.2em;
+		cursor: pointer;
+	}
+		.today {
+			font-weight: bold;
+		}
+		.day ~ .active {
+			
+			background-color: var(--main-accent);
+			color: var(--body_bg_color);
+		}
+		.day ~ .has-data {
+			outline: #07f4f4;
+			outline-style: none;
+			outline-width: medium;
+			outline-style: none;
+			outline-offset: -4px;
+			outline-style: solid;
+			outline-width: 2px;
+		}
+</style>
+
+<template>
+	<div class="squire-box">
+
+		<h3>Cycle Tracking Beta
+			<span v-on:click="deleteData()">
+				<i class="clickable trash icon"></i>
+			</span>
+		</h3>
+
+		<div class="calendar">
+			<p>{{calendar.monthName}}, {{calendar.year}}</p>
+			<div v-for="day in calendar.weekdays" class="day">
+				{{ day }}
+			</div>
+			<div v-for="day in calendar.days" class="day" 
+				:class="{
+					'today':day == calendar.today,
+					'active':dateCode == `${day}.${calendar.month}.${calendar.year}`,
+					'has-data':cycleData[`${day}.${calendar.month}.${calendar.year}`],
+				}" 
+				v-on:click="openDayData(`${day}.${calendar.month}.${calendar.year}`)">
+				{{ day }}
+			</div>
+		</div>
+
+		<div class="ui form">
+			<h3><input type="text" v-model="dateCode" v-on:keypress.enter="openDayData(dateCode)"></h3>
+			<div class="field" v-for="field in fields">
+				<label>{{ field.label }}</label>
+
+				<div v-for="(value, key) in openDay">
+
+					<!-- float -->
+					<div v-if="field.type == 'float' && field.id == key" class="ui left icon input">
+						<input type="text" :placeholder="field.label" v-on:keyup="e => saveField(field.id, e.target.value)" :value="value">
+						<i class="thermometer half icon"></i>
+					</div>
+
+					<!-- range -->
+					<div v-if="field.type == 'range' && field.id == key">
+						<div :class="{green:(value == 1)}" v-on:click="saveField(field.id, 1)" class="ui button">1</div>
+						<div :class="{green:(value == 2)}" v-on:click="saveField(field.id, 2)" class="ui button">2</div>
+						<div :class="{green:(value == 3)}" v-on:click="saveField(field.id, 3)" class="ui button">3</div>
+						<div :class="{green:(value == 4)}" v-on:click="saveField(field.id, 4)" class="ui button">4</div>
+						<div :class="{green:(value == 5)}" v-on:click="saveField(field.id, 5)" class="ui button">5</div>
+					</div>
+
+					<!-- text area -->
+					<div v-if="field.type == 'text' && field.id == key">
+						<textarea rows="3" v-on:keyup="e => saveField(field.id, e.target.value)">
+							{{ value }}
+						</textarea>
+					</div>
+
+				</div>
+
+			</div>
+
+			<div class="ui green save button" v-on:click="saveDayData">
+				Save
+			</div>
+		</div>
+
+
+	</div>
+</template>
+
+<script>
+import { defineAsyncComponent } from 'vue'
+
+	import axios from 'axios'
+
+	export default {
+		name: 'CycleTracking',
+		components: {
+			'logo':defineAsyncComponent(() => import('@/components/LogoComponent.vue')),
+		},
+		data () {
+			return {
+				appWorkingDate: null,
+				fields:[],
+				defaultFields:[
+					{'type':'float','label':'Basil Temp', 'id':'BT'},
+					{'type':'range','label':'Cervical Mucus', 'id':'CM'},
+					{'type':'text','label':'Notes', 'id':'NO'},
+				],
+				cycleData: {},
+				dateCode: null,
+				openDay: [],
+				calendar: {
+					monthName: '',
+					month: '',
+					year: '',
+					days: [],
+					weekdays: ['S','M','T','W','T','F','S'],
+					today: 0,
+				},
+			}
+		},
+		beforeCreate() {
+
+			// Perform Login check
+			this.$root.loginGateway()
+
+		},
+		mounted(){
+			if(!this.$store.getters.getLoggedIn){
+				return
+			}
+
+			// setup date code
+			// day - month - year 
+			const now = new Date()
+			this.appWorkingDate = now
+			const dateSetup = [
+				now.getDate(), // 1-31 (Day)
+				now.getMonth()+1, // 0-11 (Month)
+				now.getFullYear(), // 1888-2022 (Year)
+			]
+
+			this.dateCode = dateSetup.join('.')
+
+			this.setupCalendar(this.appWorkingDate)
+
+			this.fetchCycleData()
+		},
+		methods: {
+			saveField(fieldId, value){
+				this.openDay[fieldId] = value
+			},
+			openDayData(dateCode){
+
+				if(String(dateCode).split('.').length != 3){
+					console.log('Invalid Date Code')
+					return
+				}
+
+				this.dateCode = dateCode || this.dateCode
+
+				let currentDay = this.cycleData[this.dateCode] || {}
+
+				//Set up each entry empty or with current value
+				this.fields.forEach(field => {
+					currentDay[field.id] = currentDay[field.id] || ''
+				})
+				
+				this.openDay = currentDay
+
+			},
+			saveDayData(){
+
+				// remove empty keys
+				let cleanDayData = {}
+				Object.keys(this.openDay).forEach(key => {
+					if(this.openDay[key] != ''){
+						cleanDayData[key] = this.openDay[key]
+					}
+				})
+
+				this.cycleData[this.dateCode] = cleanDayData
+
+				// Update calendar
+				this.setupCalendar(this.appWorkingDate)
+
+				this.saveCycleData()
+
+			},
+			fetchCycleData(){
+
+				axios.post('/api/cycle-tracking/get')
+				.then(({ data }) => {
+					if(data.hasOwnProperty("text")){
+
+						let appData = null
+						try {
+							appData = JSON.parse(data.text)
+						} catch(e) {
+							console.log('Didnt parse json')
+						}
+
+						console.log(appData) 
+
+						this.cycleData = appData?.cycleData || {}
+						this.fields = appData?.fields || []
+
+						this.$nextTick(() => {
+							this.setupFields()
+							this.openDayData(this.dateCode)
+						})
+					}
+				})
+				.catch(error => { this.$bus.$emit('notification', error) })
+
+			},
+			saveCycleData(){
+
+				const appData = JSON.stringify({
+					fields: this.fields,
+					cycleData: this.cycleData,
+				})
+				axios.post('/api/cycle-tracking/save', { cycleData:appData })
+				.then(response => {
+					{ this.$bus.$emit('notification', 'Data Saved') }
+				})
+				.catch(error => { this.$bus.$emit('notification', error) })
+			},
+			deleteData(){
+				axios.post('/api/cycle-tracking/save', { cycleData:'' })
+				.then(response => {
+					{ this.$bus.$emit('notification', 'Data Deleted') }
+				})
+			},
+			setupFields(){
+
+				// push the first 3 default fields to users set
+				if(this.fields.length == 0){
+					for (let i = 0; i < 3; i++) {
+						this.fields.push(this.defaultFields[i])
+					}
+				}
+			},
+			setupCalendar(date){
+				// ------------
+				// setup calendar display
+				var y = date.getFullYear()
+				var m = date.getMonth()
+
+				var firstDay = new Date(y, m, 1);
+				var lastDay = new Date(y, m + 1, 0);
+
+				function getDaysInMonth(year, month) {
+					return new Date(year, month, 0).getDate();
+				}
+
+				const currentYear = date.getFullYear();
+				const currentMonth = date.getMonth() + 1;
+				this.calendar.monthName = date.toLocaleString("en-US", { month: "long" });
+				this.calendar.year = currentYear
+				const daysInCurrentMonth = getDaysInMonth(currentYear, currentMonth);
+
+				const monthStartDay = firstDay.getDay()
+				let days = Array(monthStartDay).fill("."); // Pad days to start on correct weekday
+				for (let i = 0; i < daysInCurrentMonth; i++) {
+					days.push(i+1)
+				}
+				this.calendar.days = days
+
+				// set today
+				this.calendar.today = date.getDate()
+				this.calendar.month = date.getMonth()+1
+
+
+				/*
+					October 2022
+				S M T W T F S
+				  1 2 3 4 5 6
+				7 8 9 
+				*/
+
+				// -------
+			}
+		}
+	}
+</script>

client/src/pages/HelpPage.vue

@@ -10,7 +10,7 @@
 
 		<div class="ui grid">
 			<div class="center aligned sixteen wide column">
-				<router-link to="/terms"></i>Solid Scribe Terms of Use</router-link>
+				<router-link to="/terms">Solid Scribe Terms of Use</router-link>
 			</div>
 		</div>
 

client/src/pages/HomePage.vue

@@ -174,7 +174,7 @@
 				</div>
 
 			<!-- Go to notes button  -->
-			<div class="row" v-if="$parent.loggedIn">
+			<div class="row" v-if="$root.loggedIn">
 				<div class="sixteen wide middle algined center aligned column">
 					<h3>You are already logged in</h3>
 					<router-link  class="ui huge green labeled icon button" to="/notes">
@@ -184,7 +184,7 @@
 			</div>
 
 			<!-- Small login form -->
-			<div class="row" v-if="!$parent.loggedIn">
+			<div class="row" v-if="!$root.loggedIn">
 				<div class="sixteen wide middle algined column">
 					<div class="ui text container">
 						<h2>
@@ -216,7 +216,7 @@
 						<h2 style="color: var(--main-accent);">
 							Pick your theme
 						</h2>
-						<h3 v-if="$parent.loggedIn">Go to settings to change theme</h3>
+						<h3 v-if="$root.loggedIn">Go to settings to change theme</h3>
 						<div 
 							v-for="color in themeColors" 
 							v-bind:key="color"
@@ -532,12 +532,13 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 export default {
 	name: 'WelcomePage',
 	components: {
-		'login-form':require('@/components/LoginFormComponent.vue').default,
+		'login-form':defineAsyncComponent(() => import('@/components/LoginFormComponent.vue')),
-		'logo':require('@/components/LogoComponent.vue').default,
+		'logo':defineAsyncComponent(() => import('@/components/LogoComponent.vue')),
-		'svg-displayer':require('@/components/SvgDisplayer.vue').default,
+		'svg-displayer':defineAsyncComponent(() => import('@/components/SvgDisplayer.vue')),
 	},
 	data(){
 		return {
@@ -562,10 +563,10 @@ export default {
 	},
 	beforeCreate(){
 		//Force HTTPS on prod, always. Dev doesn't have certs
-		const isDev = process.env['NODE_ENV'] == 'development'
+		// const isDev = process.env['NODE_ENV'] == 'development'
-		if (!isDev && location.protocol != 'https:'){
+		// if (!isDev && location.protocol != 'https:'){
-			window.location.replace('https://www.solidscribe.com')
+		// 	window.location.replace('https://www.solidscribe.com')
-		}
+		// }
 	},
 	beforeMount(){
 		

client/src/pages/LoginPage.vue

@@ -33,13 +33,14 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 	
 	import axios from 'axios';
 
 	export default {
 	name: 'Login',
 		components: {
-			'login-form':require('@/components/LoginFormComponent.vue').default,
+			'login-form':defineAsyncComponent(() => import('@/components/LoginFormComponent.vue')),
 		},
 		data () {
 			return {

client/src/pages/MetrictrackingPage.vue

@@ -355,9 +355,9 @@
 				</div>
 				
 				<div class="ui form" :class="{'loading-day':loadingDay}">
-					<draggable v-model="fields" class="ui compact grid" ghost-class="ghost" @end="onDragEnd" handle=".draggable">
+					<draggable v-model="fields" class="ui compact grid" ghost-class="ghost" @end="onDragEnd" handle=".draggable" :item-key="field => field">
-						<div v-for="field in fields" :key="field" 
+						<template #item="{ element: field }">
-							:class="userFields[field]?.width ? userFields[field]?.width+' wide stretched column':'eight wide stretched column'">
+						<div :class="userFields[field]?.width ? userFields[field]?.width+' wide stretched column':'eight wide stretched column'">
 
 							<div class="ui very compact grid input-grid"> 
 								<!-- field label display -->
@@ -443,6 +443,7 @@
 							</div>
 
 						</div>
+						</template>
 					</draggable>
 				</div>				
 			</div>
@@ -599,7 +600,6 @@
 					<div :class="{'green':(tempChartDays == 7)}" v-on:click="tempChartDays = 7; " class="ui button">7</div>
 				</div>
 				
-				<!-- <div id="graphdiv" style="width: 100%; min-height: 320px;"></div> -->
 				<div class="ui divider"></div>
 
 				<MetricGraphs 
@@ -686,8 +686,9 @@
 				<div class="ui small dividing header">
 					Active Fields
 				</div>
-				<draggable v-model="fields" class="ui very compact internally celled grid" ghost-class="ghost" @end="onDragEnd" handle=".draggable">
+				<draggable v-model="fields" class="ui very compact internally celled grid" ghost-class="ghost" @end="onDragEnd" handle=".draggable" :item-key="fieldId => fieldId">
-					<div v-for="fieldId in fields" :key="fieldId" class="row">
+					<template #item="{ element: fieldId }">
+					<div class="row">
 						<div class="eight wide column">
 							<i class="draggable grip lines icon"></i>
 							{{ userFields[fieldId]?.label }}
@@ -708,13 +709,14 @@
 							<!-- {{ openDay[fieldId] }} -->
 						</div>
 					</div>
+					</template>
 				</draggable>
 	
 				<div class="ui small dividing header">
 					Other Fields
 				</div>
-				<draggable class="ui very compact internally celled grid">
+				<div class="ui very compact internally celled grid">
-					<div class="ui row" v-for="(field,fieldId) in userFields" v-if="!fields.includes(fieldId)">
+					<div class="ui row" v-for="(field,fieldId) in unusedUserFields" :key="fieldId">
 						<div class="eight wide column">
 							<i class="draggable grip lines icon"></i>
 							{{ userFields[fieldId]?.label }}
@@ -726,8 +728,7 @@
 							<div class="ui tiny compact button" v-on:click="startUsingCustomField(fieldId)">Show</div>
 						</div>
 					</div>
-					<div class="ui row" v-for="(field,fieldId) in defaultFields" 
+					<div class="ui row" v-for="(field,fieldId) in unusedDefaultFields" :key="'default-'+fieldId">
-						v-if="!fields.includes(fieldId) && !Object.keys(userFields).includes(fieldId)">
 
 						<div class="eight wide column">
 							<i class="draggable grip lines icon"></i>
@@ -740,7 +741,7 @@
 							<div class="ui tiny compact button" v-on:click="startUsingCustomField(fieldId)">Show</div>
 						</div>
 					</div>
-				</draggable>
+				</div>
 			</div>
 
 
@@ -817,7 +818,8 @@
 					<div class="ui small dividing header">
 						Add/Edit Field
 					</div>
-					<div v-for="(entry, key) in fieldDefinition" class="row" v-if="!['id'].includes(key)">
+					<template v-for="(entry, key) in fieldDefinition" :key="key">
+					<div class="row" v-if="!['id'].includes(key)">
 						
 						<div v-if="fieldDefinitionOptionsDef[key]" class="sixteen wide column">
 							{{ mapFormTerm(key) }}
@@ -861,6 +863,7 @@
 
 						</div>
 					</div>
+					</template>
 					<div class="row">
 						<div class="eight wide column">
 							<div class="ui fluid green button" v-on:click="saveCustomField(editFieldId)" v-if="validateCustomFieldsForm()">
@@ -880,8 +883,6 @@
 			</div>
 		</modal>
 
-		<!-- may or may not need digraph CSS, maybe disect  -->
-		<!-- <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/dygraph/2.1.0/dygraph.min.css" /> -->
 
 		<div class="ui basic segment"></div>
 
@@ -890,6 +891,7 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 
 	/*
 		Important Notes:
@@ -906,12 +908,22 @@
 
 	export default {
 		name: 'MetricTracking',
+		// Descendant graph components need these; injecting avoids the
+		// $parent / $parent.$parent chains, which break whenever an
+		// intermediate instance is added (async wrappers, router-view).
+		provide() {
+			return {
+				getFieldColor: field => this.getFieldColor(field),
+				getFieldIcon:  field => this.getFieldIcon(field),
+				getFieldLabel: field => this.getFieldLabel(field),
+			}
+		},
 		components: {
-			'logo':require('@/components/LogoComponent.vue').default,
+			'logo':defineAsyncComponent(() => import('@/components/LogoComponent.vue')),
-			'delete-button': require('@/components/NoteDeleteButtonComponent.vue').default,
+			'delete-button': defineAsyncComponent(() => import('@/components/NoteDeleteButtonComponent.vue')),
-			'modal': require('@/components/ModalComponent.vue').default,
+			'modal': defineAsyncComponent(() => import('@/components/ModalComponent.vue')),
 			draggable,
-			'MetricGraphs':require('@/components/Metrictracking/MetricGraphsComponent.vue').default,
+			'MetricGraphs':defineAsyncComponent(() => import('@/components/Metrictracking/MetricGraphsComponent.vue')),
 		},
 		data () {
 			return {
@@ -991,7 +1003,7 @@
 		beforeCreate() {
 
 			// Perform Login check
-			this.$parent.loginGateway()
+			this.$root.loginGateway()
 
 		},
 		mounted(){
@@ -1020,6 +1032,21 @@
 				});
 		},
 		computed: {
+			// vuedraggable's computeNodes walks the default slot's vnodes directly
+			// and cannot handle the fragments that <template v-for> produces, so
+			// the filtering happens here instead of with a v-if in the slot.
+			unusedUserFields(){
+				return Object.fromEntries(
+					Object.entries(this.userFields).filter(([fieldId]) => !this.fields.includes(fieldId))
+				)
+			},
+			unusedDefaultFields(){
+				const userFieldIds = Object.keys(this.userFields)
+				return Object.fromEntries(
+					Object.entries(this.defaultFields).filter(([fieldId]) =>
+						!this.fields.includes(fieldId) && !userFieldIds.includes(fieldId))
+				)
+			},
 			getToday(){
 				return this.generateDateCode(new Date())
 			},
@@ -1099,6 +1126,27 @@
 				return color
 				// userFields[field]?.icon
 			},
+			getFieldLabel(field){
+
+				let label = null
+
+				// user defined
+				if(this.userFields[field]?.label){
+					label = this.userFields[field].label
+				}
+
+				// default field value
+				if(!label && this.defaultFields[field]?.label){
+					label = this.defaultFields[field]?.label
+				}
+
+				// fall back to the raw id
+				if(!label){
+					label = field
+				}
+
+				return label
+			},
 			getFieldIcon(field){
 
 				let icon = null
@@ -1218,7 +1266,7 @@
 				if(event?.target?.value){
 					option = event.target.value
 				}
-				this.$set(this.editFieldObject, key, option)
+				this.editFieldObject[key] = option
 			},
 			mapFormTerm(term){
 
@@ -1371,7 +1419,11 @@
 				if(this.openDay[fieldId] === value){ return }
 
 				// update field to be reactive
-				this.$set(this.openDay, fieldId, value)
+				this.openDay[fieldId] = value
+
+				// Push straight into cycleData so the graphs redraw immediately;
+				// the server write below stays debounced.
+				this.applyOpenDayToCycleData()
 
 				// remove debounce and set to modified
 				this.$nextTick(() => {
@@ -1401,12 +1453,14 @@
 				// open day has all fields defined, just set values
 				let currentDay = this.cycleData[dateCode] || {}
 				Object.keys(this.openDay).forEach(fieldId => {
-					this.$set(this.openDay, fieldId, currentDay[fieldId])
+					this.openDay[fieldId] = currentDay[fieldId]
 				})
 
 
 			},
-			saveDayData(){
+			// Local-only: fold the open day's values into cycleData. Cheap and
+			// synchronous, so callers can run it on every keystroke.
+			applyOpenDayToCycleData(){
 
 				// remove empty keys
 				let cleanDayData = {}
@@ -1416,12 +1470,15 @@
 					}
 				})
 
-				// Only save entry if there is data
+				// Only keep entry if there is data
 				delete this.cycleData[this.calendar.dateCode]
 				if(Object.keys(cleanDayData).length > 0){
 					this.cycleData[this.calendar.dateCode] = cleanDayData
 				}
+			},
+			saveDayData(){
 
+				this.applyOpenDayToCycleData()
 				this.saveCycleData()
 
 			},
@@ -1547,7 +1604,7 @@
 
 				// set up reactive open day object
 				Object.keys(this.userFields).forEach(fieldId => {
-					this.$set(this.openDay, fieldId, '')
+					this.openDay[fieldId] = ''
 				})
 
 				//

client/src/pages/NotesPage.vue

@@ -117,7 +117,8 @@
 					</div>
 
 					<!-- render each section based on notes in set  -->
-					<div v-for="section,index in noteSections" v-if="section.length > 0" class="note-card-section">
+					<template v-for="section,index in noteSections" :key="index">
+					<div v-if="section.length > 0" class="note-card-section">
 						<h5 class="ui tiny dividing header"><i :class="`green ${sectionData[index][0]} icon`"></i>{{ sectionData[index][1] }}</h5>
 
 						<div class="note-card-display-area">
@@ -133,6 +134,7 @@
 							/>
 						</div>
 					</div>
+					</template>
 
 
 					<div class="loading-section" v-if="showLoading">
@@ -163,20 +165,21 @@
 
 		<!-- flexbox note container evenly spaces open notes -->
 		<div class="note-panel-container" :class="{ 'note-panel-fullwidth':collapseFloatingList}" v-if="openNotes.length">
+			<template v-for="noteId in openNotes" :key="noteId">
 			<note-input-panel
-				v-for="noteId in openNotes"
 				v-if="noteId != null"
-				:key="noteId"
 				:noteid="noteId" 
 				:url-data="$route.params"
 				:open-notes="openNotes.length"
 			/>
+			</template>
 		</div>
 
 	</div>
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 	
 	import axios from 'axios'
 
@@ -184,15 +187,15 @@
 	name: 'NotesPage',
 		components: {
 
-			'note-input-panel': () => import(/* webpackChunkName: "NoteInputPanel" */ '@/components/NoteInputPanel.vue'),
+			'note-input-panel': defineAsyncComponent(() => import(/* webpackChunkName: "NoteInputPanel" */ '@/components/NoteInputPanel.vue')),
 
-			'note-title-display-card': require('@/components/NoteTitleDisplayCard.vue').default,
+			'note-title-display-card': defineAsyncComponent(() => import('@/components/NoteTitleDisplayCard.vue')),
-			// 'fast-filters': require('@/components/FastFilters.vue').default,
+			// 'fast-filters': defineAsyncComponent(() => import('@/components/FastFilters.vue')),
-			'search-input': require('@/components/SearchInput.vue').default,
+			'search-input': defineAsyncComponent(() => import('@/components/SearchInput.vue')),
-			'attachment-display': require('@/components/AttachmentDisplayCard').default,
+			'attachment-display': defineAsyncComponent(() => import('@/components/AttachmentDisplayCard.vue')),
-			'tag-display':require('@/components/TagDisplayComponent.vue').default,
+			'tag-display':defineAsyncComponent(() => import('@/components/TagDisplayComponent.vue')),
-			'loading-icon':require('@/components/LoadingIconComponent.vue').default,
+			'loading-icon':defineAsyncComponent(() => import('@/components/LoadingIconComponent.vue')),
-			'paste-button':require('@/components/PasteButton.vue').default,
+			'paste-button':defineAsyncComponent(() => import('@/components/PasteButton.vue')),
 		},
 		data () {
 			return {
@@ -203,6 +206,9 @@
 				searchTags: [],
 				notes: [],
 				openNotes: [],
+				// fullPath of our own last URL write, so the $route watcher can
+				// recognise it and not treat it as an inbound navigation
+				lastSyncedFullPath: null,
 				collapseFloatingList: false,
 				highlights: [],
 				searchDebounce: null,
@@ -258,7 +264,7 @@
 		},
 		beforeMount(){
 
-			this.$parent.loginGateway()
+			this.$root.loginGateway()
 
 			//If user is on title view, 
 			this.titleView = this.$store.getters.getIsUserOnMobile
@@ -363,7 +369,7 @@
 			// document.addEventListener('visibilitychange', this.visibiltyChangeAction);
 
 		},
-		beforeDestroy(){
+		beforeUnmount(){
 			window.removeEventListener('scroll', this.onScroll)
 			// document.removeEventListener('visibilitychange', this.visibiltyChangeAction)
 
@@ -380,11 +386,26 @@
 		},
 		mounted() {
 
-			//Open note on PAGE LOAD if ID is set
+			// Restore every note named in ?open=
-			if(this.$route.params.id > 1){
+			const ids = this.parseOpenQuery(this.$route.query.open)
-				this.openNote(this.$route.params.id)
+
+			// Fold the legacy /notes/open/:id deep link in, first position
+			const paramId = parseInt(this.$route.params.id, 10)
+			if(Number.isInteger(paramId) && paramId > 1 && !ids.includes(paramId)){
+				ids.unshift(paramId)
 			}
 
+			// Push directly rather than via openNote(), which would write the URL
+			// once per note; one normalising write follows instead.
+			ids.forEach(id => {
+				if(!this.openNotes.includes(id)){
+					this.openNotes.push(id)
+				}
+			})
+
+			// Unconditional, so a junk ?open=abc gets scrubbed too
+			this.syncOpenNotesToUrl()
+
 			//Loads initial batch and tags
 			this.reset()
 
@@ -395,17 +416,31 @@
 			},
 			'$route' (to, from) {
 
+				// Our own URL write, not an inbound navigation
+				if(to.fullPath === this.lastSyncedFullPath){ return }
 
-				// Reload the notes if returning to this page
+				// The discriminator is the absence of ?open=, NOT the path. A menu's
-				if(to.fullPath == '/notes' && !from.fullPath.includes('/notes/open/')){
+				// $router.go(-1) lands on /notes?open=5 with `from` containing
+				// '/notes/open/', so a path comparison here would close every note
+				// every time a side menu is dismissed.
+				const landingOnBareNotes = (to.path === '/notes' && !to.query.open)
+
+				// Reload the notes if arriving at an empty /notes from elsewhere
+				if(landingOnBareNotes && !from.path.startsWith('/notes/open/') && from.path !== '/notes'){
 					this.reset()
 				}
 
-				// Close all notes if returning to /notes page
+				// Explicit navigation to a note-less /notes closes everything
-				if(to.fullPath == '/notes' && from.fullPath.includes('/notes/open/')){
+				if(landingOnBareNotes && this.openNotes.length > 0){
 					this.closeAllNotes()
 				}
 
+				// Stale history entry (e.g. go(-1) after another note was closed):
+				// rewrite the URL from the array. Never read it.
+				if(to.path === '/notes' && to.query.open && to.query.open !== this.openNotes.join(',')){
+					this.syncOpenNotesToUrl()
+				}
+
 				//Lookup tags set in URL
 				if(to.params.tag && this.$store.getters.totals && this.$store.getters.totals['tags'][to.params.tag]){
 
@@ -432,81 +467,121 @@
 			}
 		},
 		methods: {
-			openNote(id, event = null){
+			// Where the open-note URL should point. A menu or a tag search owns the
+			// path, so only a plain notes view gets normalised back to /notes.
+			openNotesTargetPath(){
 
-				// 
+				// A side menu is open; its close button is a $router.go(-1), so the
+				// path has to stay put or there is nowhere sane to land.
+				if(this.$route.params.openMenu){ return this.$route.path }
 
-				const intId = parseInt(id)
+				// Preserve a tag search
-				if(this.openNotes.includes(intId)){
+				if(this.$route.params.tag){ return this.$route.path }
 
-					console.log('Open already open note?')
+				return '/notes'
+			},
+			// Mirror openNotes into ?open=. This is the ONLY place the open-note
+			// URL is written.
+			syncOpenNotesToUrl(){
 
-					// const openIndex = this.openNotes.indexOf(intId)
+				const query = Object.assign({}, this.$route.query)
-					// if(openIndex != -1){
+
-					// 	console.log('Open note and remove it ', intId + ' on index ' + openIndex)
+				if(this.openNotes.length > 0){
-					// 	this.openNotes.splice(openIndex, 1)
+					query.open = this.openNotes.join(',')
-					// }
+				}
-					// this.$bus.$emit('close_note_by_id', intId)
+				else {
+					delete query.open
+				}
+
+				const target = { path: this.openNotesTargetPath(), query }
+				const targetFullPath = this.$router.resolve(target).fullPath
+
+				// Already correct -- don't navigate at all
+				if(targetFullPath === this.$route.fullPath){
+					this.lastSyncedFullPath = targetFullPath
 					return
 				}
 
+				// Set before navigating so the watcher sees it
+				this.lastSyncedFullPath = targetFullPath
+
+				this.$router.replace(target).catch(() => {})
+			},
+			// Tolerant parse of ?open=. Handles 'abc', '1,,2,2', '-4' and the
+			// repeated form ?open=1&open=2 (which arrives as an array).
+			parseOpenQuery(raw){
+
+				if(raw === undefined || raw === null){ return [] }
+
+				// Cap: a hand-crafted URL shouldn't be able to fire 50 note fetches
+				const MAX_RESTORE = 8
+
+				return [].concat(raw)
+					.join(',')
+					.split(',')
+					.map(value => parseInt(value, 10))
+					.filter(value => Number.isInteger(value) && value > 0)
+					.filter((value, index, all) => all.indexOf(value) === index)
+					.slice(0, MAX_RESTORE)
+			},
+			openNote(id, event = null){
+
+				const intId = parseInt(id, 10)
+
+				// Also absorbs the undefined the '$route.params.id' watcher now gets
+				// when we normalise /notes/open/5 into /notes?open=5
+				if(!Number.isInteger(intId) || intId < 1){ return }
+
 				//Don't open note if a link is clicked in display card
 				if(event && event.target && event.target.nodeName){
 					const nodeClick = event.target.nodeName
 					if(nodeClick == 'A'){ return }	
 				}
 
-				// Push note to stack if not open
+				if(this.openNotes.includes(intId)){
-				if(Number.isInteger(intId) && !this.openNotes.includes(intId)){
+					// Already open, but an external push('/notes/open/5') still needs
-					this.openNotes.push(intId)
+					// the URL normalised back to the full set
+					this.syncOpenNotesToUrl()
+					return
 				}
 
-				this.$nextTick(() => {
+				this.openNotes.push(intId)
-					// change route if open ID is not the same as current ID
+				this.syncOpenNotesToUrl()
-					if(this.$route.params.id != id){
-						console.log('Open note, change route -> route id ' + this.$route.params.id + ' note id ->' + id + ', ' +(this.$route.params.id == id))
-						this.$router.push('/notes/open/'+id)
-					}
-				})
-
-				
-				
-				return
 			},
-			closeNote(noteId, modified){
+			closeNote(noteId, modified, syncUrl = true){
 
-				console.log('close note', this.$route.fullPath)
+				// parseInt: indexOf is strict, and ids arriving from the URL are
+				// strings. Without this the splice silently misses.
+				const intId = parseInt(noteId, 10)
+				const openIndex = this.openNotes.indexOf(intId)
+				const didClose = (openIndex !== -1)
 
-				const openIndex = this.openNotes.indexOf(noteId)
+				if(didClose){
-				if(openIndex != -1){
-					console.log('Removing note id ', noteId + ' on index ' + openIndex)
 					this.openNotes.splice(openIndex, 1)
 				}
 
-				// //A note has been closed
+				// Only write the URL when the array actually changed. NoteInputPanel
-				// if(this.$route.fullPath != '/notes'){
+				// emits close_active_note twice per note (closeButtonAction -> close(),
-				// 	this.$router.push('/notes')
+				// then beforeUnmount -> close()); the second arrives after the splice.
-				// }
+				if(didClose && syncUrl){
-				if(this.openNotes.length == 0 && this.$route.fullPath != '/notes'){
+					this.syncOpenNotesToUrl()
-					this.$router.push('/notes')
 				}
 
 				if(modified){
-					console.log('Just closed Note -> ' + noteId + ', modified -> ',  modified)
 					this.$store.dispatch('fetchAndUpdateUserTotals')
 					//Focus and animate if modified
 					this.updateSingleNote(noteId, modified)
 				}
-
-				console.log('closeNote(): Open notes length ', this.openNotes.length)
 			},
 			closeAllNotes(){
-				console.log('Close all notes ------------')
+
+				// Backwards, so the in-loop splice can't skip an entry. URL writes are
+				// deferred so this is one navigation rather than N.
 				for (let i = this.openNotes.length - 1; i >= 0; i--) {
-					console.log('Close all notes -> ' + this.openNotes[i])
+					this.closeNote(this.openNotes[i], false, false)
-					this.closeNote(this.openNotes[i])
 				}
-				console.log('----------------')
+
+				this.syncOpenNotesToUrl()
 			},
 			toggleTagFilter(tagId){
 
@@ -741,7 +816,10 @@
 
 						return resolve(true)
 					})
-					.catch(error => { this.$bus.$emit('notification', 'Failed to Search Notes') })
+					.catch(error => {
+						console.error('[NotesPage] search failed:', error)
+						this.$bus.$emit('notification', 'Failed to Search Notes')
+					})
 				})
 			},
 			rebuildNoteCategorise(){

client/src/pages/OverviewPage.vue

@@ -106,7 +106,8 @@
 				<div :class="{'one-column':( showOneColumn() )}">
 
 					<!-- render each section based on notes in set  -->
-					<div v-for="section,index in noteSections" v-if="section.length > 0" class="note-card-section">
+					<template v-for="section,index in noteSections" :key="index">
+					<div v-if="section.length > 0" class="note-card-section">
 						<h5 class="ui tiny dividing header"><i :class="`green ${sectionData[index][0]} icon`"></i>{{ sectionData[index][1] }}</h5>
 
 						<div class="note-card-display-area">
@@ -122,6 +123,7 @@
 							/>
 						</div>
 					</div>
+					</template>
 
 
 					<loading-icon v-if="loadingInProgress" message="Decrypting Notes" />
@@ -143,6 +145,7 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 	
 	import axios from 'axios'
 
@@ -150,15 +153,15 @@
 	name: 'SearchBar',
 		components: {
 
-			'note-input-panel': () => import(/* webpackChunkName: "NoteInputPanel" */ '@/components/NoteInputPanel.vue'),
+			'note-input-panel': defineAsyncComponent(() => import(/* webpackChunkName: "NoteInputPanel" */ '@/components/NoteInputPanel.vue')),
 
-			'note-title-display-card': require('@/components/NoteTitleDisplayCard.vue').default,
+			'note-title-display-card': defineAsyncComponent(() => import('@/components/NoteTitleDisplayCard.vue')),
-			// 'fast-filters': require('@/components/FastFilters.vue').default,
+			// 'fast-filters': defineAsyncComponent(() => import('@/components/FastFilters.vue')),
-			'search-input': require('@/components/SearchInput.vue').default,
+			'search-input': defineAsyncComponent(() => import('@/components/SearchInput.vue')),
-			'attachment-display': require('@/components/AttachmentDisplayCard').default,
+			'attachment-display': defineAsyncComponent(() => import('@/components/AttachmentDisplayCard.vue')),
-			'counter':require('@/components/AnimatedCounterComponent.vue').default,
+			'counter':defineAsyncComponent(() => import('@/components/AnimatedCounterComponent.vue')),
-			'tag-display':require('@/components/TagDisplayComponent.vue').default,
+			'tag-display':defineAsyncComponent(() => import('@/components/TagDisplayComponent.vue')),
-			'loading-icon':require('@/components/LoadingIconComponent.vue').default,
+			'loading-icon':defineAsyncComponent(() => import('@/components/LoadingIconComponent.vue')),
 		},
 		data () {
 			return {
@@ -222,7 +225,7 @@
 		},
 		beforeMount(){
 
-			this.$parent.loginGateway()
+			this.$root.loginGateway()
 
 			this.$io.on('new_note_created', noteId => {
 
@@ -335,7 +338,7 @@
 			// document.addEventListener('visibilitychange', this.visibiltyChangeAction);
 
 		},
-		beforeDestroy(){
+		beforeUnmount(){
 			window.removeEventListener('scroll', this.onScroll)
 			// document.removeEventListener('visibilitychange', this.visibiltyChangeAction)
 

client/src/pages/QuickPage.vue

@@ -86,7 +86,7 @@
 			//
 			// Perform Login check
 			//
-			this.$parent.loginGateway()
+			this.$root.loginGateway()
 		},
 		beforeMount(){
 

client/src/pages/SettingsPage.vue

@@ -44,6 +44,42 @@
 			</div>
 		</div>
 
+		<!-- App / PWA -->
+		<h4 class="ui header">
+			App
+		</h4>
+		<div class="ui segment">
+			<div class="ui doubling grid">
+				<div class="sixteen wide column">
+
+					<p>
+						Which page the installed app opens to. Set automatically to
+						whichever page you installed from, and changeable here.
+					</p>
+					<div
+						v-for="option in launchOptions"
+						:key="option.path"
+						class="ui compact basic button"
+						:class="{'green':(launchPage == option.path)}"
+						v-on:click="chooseLaunchPage(option.path)">
+						{{ option.label }}
+					</div>
+
+					<div class="ui divider"></div>
+
+					<p>
+						If the app gets stuck on an old version, this clears the stored
+						copies of its files and reloads. Your notes are not affected.
+					</p>
+					<div class="ui compact basic button" v-on:click="clearAppCache()">
+						<i class="redo icon"></i>
+						Reset App Cache
+					</div>
+
+				</div>
+			</div>
+		</div>
+
 		<!-- Enable Two Factor -->
 		<h4>Two Factor Authentication</h4>
 		<div class="ui segment">
@@ -152,7 +188,7 @@
 
 		<div class="ui grid">
 			<div class="center aligned sixteen wide column">
-				<router-link to="/terms"></i>Solid Scribe Terms of Use</router-link>
+				<router-link to="/terms">Solid Scribe Terms of Use</router-link>
 			</div>
 		</div>
 		
@@ -161,13 +197,17 @@
 </template>
 
 <script>
+import { defineAsyncComponent } from 'vue'
 
 	import axios from 'axios'
 
+	import { LAUNCH_OPTIONS, getLaunchPage, setLaunchPage } from '@/LaunchPage'
+	import { resetAppCache } from '@/registerServiceWorker'
+
 	export default {
 		name: 'SettingsPage',
 		components: {
-			'logo':require('@/components/LogoComponent.vue').default,
+			'logo':defineAsyncComponent(() => import('@/components/LogoComponent.vue')),
 		},
 		data () {
 			return {
@@ -176,6 +216,8 @@
 				verificationToken: '',
 				showNewNoteConfirm: false,
 
+				launchOptions: LAUNCH_OPTIONS,
+				launchPage: getLaunchPage(),
 				themeColors: [
 					'#21BA45', //Green
 					'#b5cc18', //Lime
@@ -197,6 +239,14 @@
 			}
 		},
 		methods: {
+			chooseLaunchPage(path){
+				setLaunchPage(path)
+				this.launchPage = getLaunchPage()
+				this.$bus.$emit('notification', 'App will open to ' + path)
+			},
+			clearAppCache(){
+				resetAppCache()
+			},
 			newQuickNote(){
 
 				this.showNewNoteConfirm = true

client/src/pages/ShareTargetPage.vue

@@ -0,0 +1,148 @@
+<template>
+	<div class="ui container share-target">
+
+		<div class="ui basic center aligned segment">
+
+			<div v-if="status === 'working'">
+				<loading-icon message="Saving to Scratch Pad" />
+			</div>
+
+			<div v-else-if="status === 'error'">
+				<h3 class="ui header">
+					<i class="exclamation triangle icon"></i>
+					Couldn't save that
+				</h3>
+				<p>You may be offline. Your text is safe below &mdash; retry when you have a connection.</p>
+
+				<textarea class="shared-text" rows="8" readonly :value="pushText"></textarea>
+
+				<div class="ui buttons">
+					<div class="ui green button" v-on:click="save()">
+						<i class="redo icon"></i> Retry
+					</div>
+					<div class="ui basic button" v-on:click="discard()">
+						Discard
+					</div>
+				</div>
+			</div>
+
+			<div v-else>
+				<loading-icon message="Working" />
+			</div>
+
+		</div>
+	</div>
+</template>
+
+<script>
+	import { defineAsyncComponent } from 'vue'
+	import axios from 'axios'
+
+	// Shared text is parked here when the share arrives while logged out. The
+	// share's query string does not survive the bounce through /login, so a
+	// redirect param alone would lose it.
+	const STASH_KEY = 'pendingShareText'
+
+	// Guard against a pathological share wedging localStorage
+	const MAX_STASH = 100 * 1024
+
+	export default {
+		name: 'ShareTargetPage',
+		components: {
+			'loading-icon': defineAsyncComponent(() => import('@/components/LoadingIconComponent.vue')),
+		},
+		data(){
+			return {
+				status: 'working',
+				pushText: '',
+			}
+		},
+		// Deliberately NOT calling $root.loginGateway() the way other pages do --
+		// it would bounce away and discard the shared payload before we read it.
+		mounted(){
+
+			const text = this.composeSharedText()
+
+			if(!text){
+				this.$router.replace('/quick')
+				return
+			}
+
+			this.pushText = text
+
+			// getLoggedIn is reliable here: App.vue's beforeCreate populates the
+			// username from localStorage before any route component mounts.
+			if(!this.$store.getters.getLoggedIn){
+				this.stash(text)
+				this.$router.replace({ path: '/login', query: { redirect: '/share-target' } })
+				return
+			}
+
+			this.save()
+		},
+		methods: {
+			// Android routinely duplicates content across these fields -- the URL
+			// lands in both `text` and `url`, the page title in both `title` and
+			// `text`. Without the dedupe every shared link is saved twice.
+			composeSharedText(){
+
+				const query = this.$route.query
+
+				const parts = ['title', 'text', 'url']
+					.map(key => String(query[key] ?? '').trim())
+					.filter(value => value.length > 0)
+					.filter((value, index, all) => all.indexOf(value) === index)
+
+				if(parts.length > 0){ return parts.join('\n') }
+
+				// Nothing in the URL: this is the replay after logging in
+				return localStorage.getItem(STASH_KEY) || ''
+			},
+			stash(text){
+				if(text.length > MAX_STASH){ return }
+				localStorage.setItem(STASH_KEY, text)
+			},
+			save(){
+
+				this.status = 'working'
+
+				// Same call the paste button makes. The server prepends to the
+				// scratch pad, strips HTML and auto-links URLs, so nothing needs
+				// formatting here.
+				axios.post('/api/quick-note/update', { 'pushText': this.pushText })
+				.then(() => {
+					localStorage.removeItem(STASH_KEY)
+					this.$bus.$emit('notification', 'Saved To Scratch Pad')
+					this.$router.replace('/quick')
+				})
+				.catch(error => {
+					// Keep the stash: offline is the likely cause and the service
+					// worker means the app boots without a connection.
+					console.error('[ShareTarget] save failed:', error)
+					this.stash(this.pushText)
+					this.status = 'error'
+				})
+			},
+			discard(){
+				localStorage.removeItem(STASH_KEY)
+				this.$router.replace('/quick')
+			},
+		},
+	}
+</script>
+
+<style scoped>
+	.share-target {
+		padding-top: 3em;
+	}
+	.shared-text {
+		width: 100%;
+		margin: 1em 0;
+		padding: 0.75em;
+		border: 1px solid var(--border_color);
+		border-radius: 4px;
+		background-color: var(--small_element_bg_color);
+		color: var(--text_color);
+		font-family: inherit;
+	}
+</style>

client/src/registerServiceWorker.js

@@ -0,0 +1,48 @@
+import { registerSW } from 'virtual:pwa-register'
+import EventBus from './EventBus'
+
+// How often to ask the server whether a new build exists
+const UPDATE_INTERVAL = 60 * 60 * 1000
+
+//
+// The worker precaches static build output only -- see the SECURITY INVARIANT
+// comment in vite.config.js. Nothing user-derived ever enters Cache Storage,
+// which is why caches are deliberately NOT cleared on logout: doing so would
+// only destroy the offline shell.
+//
+export const updateSW = registerSW({
+	onNeedRefresh(){
+		// A new build is waiting. Let the user apply it rather than swapping
+		// assets underneath them mid-edit.
+		EventBus.$emit('sw-update-available')
+	},
+	onOfflineReady(){
+		EventBus.$emit('notification', 'Ready to work offline')
+	},
+	onRegisteredSW(swUrl, registration){
+		if(!registration){ return }
+		setInterval(() => registration.update(), UPDATE_INTERVAL)
+	},
+	onRegisterError(error){
+		console.error('[SW] registration failed:', error)
+	},
+})
+
+//
+// Escape hatch. A wedged service worker is very hard to talk someone through
+// remotely, so this is surfaced as a button in Settings.
+//
+export async function resetAppCache(){
+
+	if('serviceWorker' in navigator){
+		const registrations = await navigator.serviceWorker.getRegistrations()
+		await Promise.all(registrations.map(registration => registration.unregister()))
+	}
+
+	if('caches' in window){
+		const keys = await caches.keys()
+		await Promise.all(keys.map(key => caches.delete(key)))
+	}
+
+	window.location.reload()
+}

client/src/router/index.js

@@ -1,133 +1,41 @@
-import Vue from 'vue'
+import { createRouter, createWebHistory } from 'vue-router'
-import Router from 'vue-router'
 
-//Breaking components into function sections allows webpack to load them dynamically
+const HomePage          = () => import('@/pages/HomePage.vue')
+const LoginPage         = () => import('@/pages/LoginPage.vue')
+const HelpPage          = () => import('@/pages/HelpPage.vue')
+const TermsPage         = () => import('@/pages/TermsPage.vue')
+const SettingsPage      = () => import('@/pages/SettingsPage.vue')
+const SharePage         = () => import('@/pages/SharePage.vue')
+const NotesPage         = () => import('@/pages/NotesPage.vue')
+const QuickPage         = () => import('@/pages/QuickPage.vue')
+const AttachmentsPage   = () => import('@/pages/AttachmentsPage.vue')
+const OverviewPage      = () => import('@/pages/OverviewPage.vue')
+const BookmarkletPage   = () => import('@/pages/BookmarkletPage.vue')
+const NotFoundPage      = () => import('@/pages/NotFoundPage.vue')
+const MetrictrackingPage = () => import('@/pages/MetrictrackingPage.vue')
+const ShareTargetPage   = () => import('@/pages/ShareTargetPage.vue')
 
-const HomePage = () => import(/* webpackChunkName: "HomePage" */ '@/pages/HomePage')
+export default createRouter({
-const LoginPage = () => import(/* webpackChunkName: "LoginPage" */ '@/pages/LoginPage')
+	history: createWebHistory(),
-const HelpPage = () => import(/* webpackChunkName: "HelpPage" */ '@/pages/HelpPage')
+	routes: [
-const TermsPage = () => import(/* webpackChunkName: "TermsPage" */ '@/pages/TermsPage')
+		{ path: '/',                            name: 'HomePage',              meta: { title: 'Home' },                  component: HomePage },
-const SettingsPage = () => import(/* webpackChunkName: "SettingsPage" */ '@/pages/SettingsPage')
+		{ path: '/login',                       name: 'LoginPage',             meta: { title: 'Login' },                 component: LoginPage },
-const SharePage = () => import(/* webpackChunkName: "SharePage" */ '@/pages/SharePage')
+		{ path: '/notes',                       name: 'Note Page',             meta: { title: 'Notes' },                 component: NotesPage },
-const NotesPage = () => import(/* webpackChunkName: "NotesPage" */ '@/pages/NotesPage')
+		{ path: '/notes/open/:id',              name: 'Open Note',             meta: { title: 'Open Note' },             component: NotesPage },
-const QuickPage = () => import(/* webpackChunkName: "QuickPage" */ '@/pages/QuickPage')
+		{ path: '/search/tags/:tag',            name: 'Search Notes',          meta: { title: 'Search Notes' },          component: NotesPage },
-const AttachmentsPage = () => import(/* webpackChunkName: "AttachmentsPage" */ '@/pages/AttachmentsPage')
+		{ path: '/notes/open/:id/menu/:openMenu', name: 'Open Note Menu',      meta: { title: 'Open Note Menu' },        component: NotesPage },
-const OverviewPage = () => import(/* webpackChunkName: "OverviewPage" */ '@/pages/OverviewPage')
+		{ path: '/help',                        name: 'Help',                  meta: { title: 'Help' },                  component: HelpPage },
-const BookmarkletPage = () => import(/* webpackChunkName: "BookmarkletPage" */ '@/pages/BookmarkletPage')
+		{ path: '/terms',                       name: 'Terms',                 meta: { title: 'Terms' },                 component: TermsPage },
-const NotFoundPage = () => import(/* webpackChunkName: "404Page" */ '@/pages/NotFoundPage')
+		{ path: '/bookmarklet',                 name: 'Bookmarklet',           meta: { title: 'Bookmarklet' },           component: BookmarkletPage },
-
+		{ path: '/settings',                    name: 'Settings',              meta: { title: 'Settings' },              component: SettingsPage },
-Vue.use(Router)
+		{ path: '/public/note/:id/:token',      name: 'Share',                 meta: { title: 'Shared' },                component: SharePage },
-
+		{ path: '/share-target',                name: 'Share Target',          meta: { title: 'Saving Share' },          component: ShareTargetPage },
-export default new Router({
+		{ path: '/quick',                       name: 'Quick',                 meta: { title: 'Scratch Pad' },           component: QuickPage },
-  routes: [
+		{ path: '/attachments',                 name: 'Attachments',           meta: { title: 'Attachments' },           component: AttachmentsPage },
-    {
+		{ path: '/attachments/note/:id',        name: 'Attachments for Note',  meta: { title: 'Attachments for Note' },  component: AttachmentsPage },
-      path: '/',
+		{ path: '/attachments/type/:type',      name: 'Attachments by Type',   meta: { title: 'Attachments by Type' },   component: AttachmentsPage },
-      name: 'HomePage',
+		{ path: '/overview',                    name: 'Overview of Notes',     meta: { title: 'Overview of Notes' },     component: OverviewPage },
-      meta: {title:'Home'},
+		{ path: '/metrictrack',                 name: 'Metric Tracking',       meta: { title: 'Metric Tracking' },       component: MetrictrackingPage },
-      component: HomePage
+		{ path: '/:pathMatch(.*)*',             name: 'Page Not Found',        meta: { title: '404 Page Not Found' },    component: NotFoundPage },
-    },
+	],
-    {
-      path: '/login',
-      name: 'LoginPage',
-      meta: {title:'Login'},
-      component: LoginPage
-    },
-    {
-      path: '/notes',
-      name: 'Note Page', //don't change this
-      meta: {title:'Notes'},
-      component: NotesPage
-    },
-    {
-      path: '/notes/open/:id',
-      name: 'Open Note',
-      meta: {title: 'Open Note'},
-      component: NotesPage,
-    },
-    {
-      path: '/search/tags/:tag',
-      name: 'Search Notes',
-      meta: {title: 'Search Notes'},
-      component: NotesPage,
-    },
-    {
-      path: '/notes/open/:id/menu/:openMenu',
-      name: 'Open Note Menu',
-      meta: {title: 'Open Note Menu'},
-      component: NotesPage,
-    },
-    {
-      path: '/help',
-      name: 'Help',
-      meta: {title:'Help'},
-      component: HelpPage
-    },
-    {
-      path: '/terms',
-      name: 'Terms',
-      meta: {title:'Terms'},
-      component: TermsPage
-    },
-    {
-      path: '/bookmarklet',
-      name: 'Bookmarklet',
-      meta: {title:'Bookmarklet'},
-      component: BookmarkletPage
-    },
-    {
-      path: '/settings',
-      name: 'Settings',
-      meta: {title:'Settings'},
-      component: SettingsPage
-    },
-    {
-      path: '/public/note/:id/:token',
-      name: 'Share',
-      meta: {title:'Shared'},
-      component: SharePage
-    },
-    {
-      path: '/quick',
-      name: 'Quick',
-      meta: {title:'Scratch Pad'},
-      component: QuickPage
-    },
-    {
-      path: '/attachments',
-      name: 'Attachments',
-      meta: {title:'Attachments'},
-      component: AttachmentsPage
-    },
-    {
-      path: '/attachments/note/:id',
-      name: 'Attachments for Note',
-      meta: {title:'Attachments for Note'},
-      component: AttachmentsPage
-    },
-    {
-      path: '/attachments/type/:type',
-      name: 'Attachments by Type',
-      meta: {title:'Attachments by Type'},
-      component: AttachmentsPage
-    },
-    {
-      path: '/overview',
-      name: 'Overview of Notes',
-      meta: {title:'Overview of Notes'},
-      component: OverviewPage
-    },
-    {
-      path: '*',
-      name: 'Page Not Found',
-      meta: {title:'404 Page Not Found'},
-      component: NotFoundPage
-    },
-    // Cycle Tracking
-    {
-      path: '/metrictrack',
-      name: 'Metric Tracking',
-      meta: {title:'Metric Tracking'},
-      component: () => import(/* webpackChunkName: "MetrictrackingPage" */ '@/pages/MetrictrackingPage')
-    },
-  ]
 })

client/src/store/index.js

@@ -1,15 +0,0 @@
-import Vue from 'vue'
-import Vuex from 'vuex'
-
-Vue.use(Vuex)
-
-export default new Vuex.Store({
-  state: {
-  },
-  mutations: {
-  },
-  actions: {
-  },
-  modules: {
-  }
-})

client/src/stores/mainStore.js

@@ -1,10 +1,7 @@
-import Vue from 'vue'
+import { createStore } from 'vuex'
-import Vuex from 'vuex'
 import axios from 'axios'
 
-Vue.use(Vuex);
+export default createStore({
-
-export default new Vuex.Store({
 	state: {
 		username: null,
 		nightMode: false,

client/src/style.css

@@ -0,0 +1,79 @@
+:root {
+  font-family: system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+.card {
+  padding: 2em;
+}
+
+#app {
+  max-width: 1280px;
+  margin: 0 auto;
+  padding: 2rem;
+  text-align: center;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

client/src/tailwind.css

@@ -0,0 +1,25 @@
+/*
+ * Tailwind 4, wired to coexist with Fomantic UI.
+ *
+ * Deliberately NOT `@import "tailwindcss"` (the whole thing). That pulls in
+ * Preflight as `@layer base`, which zeroes margin/padding/border on every
+ * element, strips heading sizes and list markers, forces img/svg to
+ * display:block and makes buttons and inputs transparent. Dropped on top of
+ * Fomantic it visibly breaks the existing UI.
+ *
+ * Utilities are imported UNLAYERED on purpose. Tailwind's own index.css wraps
+ * them in `@layer utilities`, and an unlayered stylesheet beats any layered
+ * rule regardless of specificity -- since all of Fomantic is unlayered, the
+ * utilities would quietly lose every conflict. Unlayered they compete on normal
+ * specificity instead, and win ties by source order because this file is
+ * imported last in main.js.
+ *
+ * Fomantic mostly uses two-class selectors (.ui.button = specificity 0,2,0), so
+ * it still out-specifies a single utility (0,1,0). To beat Fomantic on a
+ * property it already sets, use the important modifier -- `p-4!`, `bg-red-500!`.
+ *
+ * To try Preflight anyway, add:  @import "tailwindcss/preflight.css" layer(base);
+ */
+
+@import "tailwindcss/theme.css" layer(theme);
+@import "tailwindcss/utilities.css";

client/vite.config.js

@@ -0,0 +1,114 @@
+import fs from 'node:fs'
+import { defineConfig } from 'vite'
+import vue from '@vitejs/plugin-vue'
+import tailwindcss from '@tailwindcss/vite'
+import { VitePWA } from 'vite-plugin-pwa'
+import { fileURLToPath, URL } from 'node:url'
+
+// Opt-in HTTPS for the dev server, off by default so the usual
+// http://localhost:5173 workflow is unchanged. Only needed if you want to test
+// the installed PWA from a phone over the LAN *without* using
+// chrome://flags/#unsafely-treat-insecure-origin-as-secure -- and note Chrome
+// needs the mkcert root CA installed on the device for the origin to be
+// service-worker eligible; clicking through a cert warning is not enough.
+//   DEV_HTTPS=1 pnpm dev
+const certDir = fileURLToPath(new URL('./certs', import.meta.url))
+const keyPath = `${certDir}/192.168.1.164+4-key.pem`
+const certPath = `${certDir}/192.168.1.164+4.pem`
+const useHttps =
+  process.env.DEV_HTTPS === '1' && fs.existsSync(keyPath) && fs.existsSync(certPath)
+
+export default defineConfig({
+  plugins: [
+    vue({
+      template: {
+        transformAssetUrls: { includeAbsolute: false },
+      },
+    }),
+    tailwindcss(),
+    VitePWA({
+      // Updates are user-driven via the prompt in src/registerServiceWorker.js
+      registerType: 'prompt',
+      injectRegister: false,
+      // Lands at dist/sw.js -- root of the served directory, so the worker gets
+      // root scope without needing a Service-Worker-Allowed header.
+      filename: 'sw.js',
+      // We ship public/manifest.json by hand rather than generating one.
+      manifest: false,
+      workbox: {
+        //
+        // SECURITY INVARIANT -- do not relax without thinking it through.
+        //
+        // Solid Scribe is end-to-end encrypted. The service worker precaches
+        // ONLY static build output and is network-only for /api/*. Caching an
+        // API response would write decrypted note content into Cache Storage on
+        // disk, where it survives logout and sits outside the encryption model.
+        //
+        // Structurally this holds because globPatterns only ever reads dist/,
+        // and user content is served by Express from staticFiles/ at
+        // /api/static -- which never enters dist. The NetworkOnly rule below is
+        // belt-and-braces: it documents the invariant and stops a future
+        // runtimeCaching entry from silently widening the net.
+        //
+        // woff2 is NOT in Workbox's default globPatterns. Without it the Roboto
+        // files are skipped and the offline shell renders in a fallback face.
+        globPatterns: ['**/*.{js,css,html,woff2,png,svg,ico,json}'],
+        // Vendor chunks (fomantic, chart.js) exceed the 2MiB default. An
+        // oversized file is silently OMITTED from the precache, which half
+        // breaks the offline shell -- watch the build output for a warning.
+        maximumFileSizeToCacheInBytes: 5 * 1024 * 1024,
+        navigateFallback: '/index.html',
+        // Matters for top-level navigations to /api/static/<file> (clicking an
+        // attachment link) -- without this the worker answers with index.html.
+        navigateFallbackDenylist: [/^\/api\//, /^\/socket/],
+        cleanupOutdatedCaches: true,
+        // Let the user decide when to swap assets, rather than doing it
+        // mid-session underneath them.
+        clientsClaim: false,
+        skipWaiting: false,
+        //
+        // runtimeCaching is deliberately EMPTY. Do not add an entry for /api/*
+        // or /socket -- not even NetworkOnly.
+        //
+        // There used to be a NetworkOnly route for both, as belt-and-braces for
+        // the invariant above. It was worse than useless. registerRoute() makes
+        // the worker call event.respondWith() for every matching request, and
+        // Socket.IO's long-poll requests are aborted as a matter of course --
+        // on reconnect, on transport upgrade, and whenever Android backgrounds
+        // the page. An aborted fetch inside NetworkOnly throws Workbox's
+        // `no-response`, the respondWith promise rejects, and Socket.IO sees a
+        // dead transport and reconnects. The installed PWA was dropping its
+        // live note sync on a loop and filling the logs with
+        // "FetchEvent ... resulted in a network error response".
+        //
+        // With no route registered the worker never intercepts these at all and
+        // the browser handles them natively, exactly as in a normal tab.
+        //
+        // Nothing is lost on the security side: the invariant is enforced by
+        // globPatterns only ever reading dist/, and by there being no *caching*
+        // strategy here. NetworkOnly enforced nothing -- it only inserted the
+        // worker into the hot path of every poll.
+        //
+      },
+      // Flip to true temporarily to exercise the worker on the dev server.
+      // Remember to unregister it afterwards -- a lingering dev service worker
+      // is a classic "why is my phone showing an ancient build".
+      devOptions: { enabled: false, type: 'module' },
+    }),
+  ],
+  resolve: {
+    alias: {
+      '@': fileURLToPath(new URL('./src', import.meta.url)),
+    },
+  },
+  server: {
+    host: '0.0.0.0',
+    ...(useHttps
+      ? { https: { key: fs.readFileSync(keyPath), cert: fs.readFileSync(certPath) } }
+      : {}),
+    proxy: {
+      '/api': 'http://localhost:3000',
+      '/socket': { target: 'http://localhost:3001', ws: true },
+    },
+  },
+})

configs/dev nginx sites available default.cfg

@@ -0,0 +1,97 @@
+#
+# Working dev server config
+#
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name 192.168.1.164;
+    return 301 https://$host$request_uri;
+}
+
+
+server {
+    
+    listen 443 ssl;
+
+    ssl_certificate /home/mab/ss/client/certs/nginx-selfsigned.crt;
+    ssl_certificate_key /home/mab/ss/client/certs/nginx-selfsigned.key;
+    ssl_dhparam /home/mab/ss/client/certs/dhparam.pem;
+
+    ssl_session_cache    shared:SSL:1m;
+    ssl_session_timeout  5m;
+    ssl_protocols        TLSV1.1 TLSV1.2 TLSV1.3;
+ 
+    ssl_ciphers  HIGH:!aNULL:!MD5;
+    ssl_prefer_server_ciphers  on;
+
+    access_log /var/log/nginx/httpslocalhost.access.log;
+    error_log  /var/log/nginx/httpslocalhost.error.log;
+
+    client_max_body_size 20M;
+
+    location / {
+            proxy_pass  https://127.0.0.1:8081;
+            proxy_set_header    Host                localhost;
+            proxy_set_header    X-Forwarded-Host    localhost;
+            proxy_set_header    X-Forwarded-Server  localhost;
+            proxy_set_header    X-Forwarded-Proto   $scheme;
+            proxy_set_header    X-Real-IP           $remote_addr;
+            proxy_set_header    X-Forwarded-For     $remote_addr;
+            proxy_redirect off;
+            proxy_connect_timeout 90s;
+            proxy_read_timeout 90s;
+            proxy_send_timeout 90s;
+            proxy_ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    }
+
+    location /sockjs-node {
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-NginX-Proxy true;
+
+        proxy_pass https://127.0.0.1:8081;
+        proxy_redirect off;
+        proxy_ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    }
+
+    location /api {
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-NginX-Proxy true;
+
+            proxy_pass http://127.0.0.1:3000;
+            proxy_redirect off;
+            proxy_ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    }
+
+    location /socket {
+        proxy_pass http://127.0.0.1:3001;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+        proxy_set_header Host $host;
+    }
+
+}
+
+
+
+# Prod settings to serve static index
+#    location / {
+#        autoindex on;
+#       #try_files $uri $uri/ /index.html;
+#    }
+
+#    location / {
+#       #autoindex on
+#
+#        proxy_pass http://127.0.0.1:8444;
+#        proxy_http_version 1.1;
+#        proxy_set_header Upgrade $http_upgrade;
+#        proxy_set_header Connection 'upgrade';
+#        proxy_set_header Host $host;
+#        proxy_cache_bypass $http_upgrade;
+#     }

configs/nginx-nixos.nix

@@ -0,0 +1,103 @@
+#
+# nginx for Solid Scribe on NixOS.
+#
+# Merge this into your configuration.nix and `nixos-rebuild switch`.
+# Replace REPO_PATH with the absolute path of the repo clone on the server.
+#
+# Four things differ from the previous config, and the first two are required
+# for the Vue 3 client to work at all:
+#
+#   1. SPA fallback. The old Vue 2 client used vue-router's default HASH mode
+#      (no `mode:` was set), so every request was just "/" and routing happened
+#      after the "#". That is why share links looked like /#/public/note/...
+#      The Vue 3 client uses createWebHistory, so /notes, /settings and
+#      /share-target are real HTTP requests. Without tryFiles they all 404.
+#
+#   2. root moves up to the vhost. It was inside locations."/", which left
+#      /assets/, /sw.js and /manifest.json unresolvable once they needed their
+#      own location blocks.
+#
+#      It also no longer points into the repo. deploy.sh publishes the built
+#      client to /var/www/ss/releases/<timestamp> and atomically repoints the
+#      /var/www/ss/current symlink at it. Pointing nginx at that stable path
+#      means a deploy is one rename -- nginx resolves the symlink per request,
+#      so no reload is needed and no request sees a half-copied directory.
+#      Rollback is the same rename pointing at an older release.
+#
+#   3. proxyWebsockets on /socket. recommendedProxySettings does NOT add the
+#      Upgrade/Connection headers, so Socket.IO has been silently falling back
+#      to long-polling -- that's the live note-sync path running degraded.
+#
+#   4. Cache headers. See the warning below.
+#
+# !! CLOUDFLARE !!
+# The tunnel sits in front of this and Cloudflare caches .js by DEFAULT, which
+# includes /sw.js. If the edge caches the service worker, the update check can
+# never see a new one and the app is pinned to an old build with no remote way
+# to recover. The nginx headers below are necessary but NOT sufficient --
+# add a Cloudflare Cache Rule that BYPASSES cache for /sw.js (and /index.html)
+# before cutting over.
+#
+# Also confirm which Host header cloudflared sends. This vhost is keyed on
+# "192.168.1.7"; if the tunnel forwards the public hostname instead, nginx will
+# not match this vhost and will fall through to the default server. Either add
+# the public hostname to serverAliases or set the tunnel's httpHostHeader.
+#
+
+services.nginx = {
+  enable = true;
+  recommendedProxySettings = true;
+  recommendedGzipSettings = true;
+
+  virtualHosts."192.168.1.7" = {
+    # serverAliases = [ "notes.example.com" ];   # if the tunnel forwards the public host
+
+    # Published by deploy.sh; a symlink to the live release. Not in the repo.
+    root = "/var/www/ss/current";
+
+    locations."/" = {
+      tryFiles = "$uri $uri/ /index.html";
+    };
+
+    # Never let the HTTP layer cache the service worker or the app shell.
+    locations."= /sw.js".extraConfig = ''
+      add_header Cache-Control "no-cache, no-store, must-revalidate";
+      expires -1;
+    '';
+
+    locations."= /index.html".extraConfig = ''
+      add_header Cache-Control "no-cache, no-store, must-revalidate";
+      expires -1;
+    '';
+
+    # The manifest changes rarely but should not be pinned for a year.
+    locations."= /manifest.json".extraConfig = ''
+      add_header Cache-Control "no-cache";
+      expires -1;
+    '';
+
+    # Vite content-hashes everything under /assets, so these are immutable.
+    locations."/assets/".extraConfig = ''
+      add_header Cache-Control "public, max-age=31536000, immutable";
+    '';
+
+    locations."/api" = {
+      proxyPass = "http://localhost:3000";
+    };
+
+    locations."/socket" = {
+      proxyPass = "http://localhost:3001";
+      proxyWebsockets = true;
+    };
+  };
+};
+
+# Note on file access: nginx runs as its own user and needs read access to the
+# web root, with +x on every parent directory. Serving from /var/www/ss is what
+# makes that a non-issue -- the one-time `chmod 755 /var/www /var/www/ss` in
+# deploy.sh is the whole of it. This is the reason the root is not the repo
+# clone: a clone under /home/mab would require loosening the permissions on the
+# home directory itself.
+#
+# Uploads are NOT here. Express serves them from REPO_PATH/staticFiles at
+# /api/static via the proxy below, so they are untouched by deploys.

deploy.sh

@@ -0,0 +1,225 @@
+#!/usr/bin/env bash
+#
+# Deploy Solid Scribe. Run this ON the server, from inside the repo.
+#
+#   cd /path/to/SolidScribe && ./deploy.sh
+#
+# Replaces buildAndUpdateProd.sh, which built the old Vue 2 client and rsynced a
+# tarball to /home/mab/pi -- a directory that no longer exists.
+#
+# Assumes:
+#   - this repo is a git clone with a tracking branch
+#   - node + pnpm are on PATH for the user running this (on NixOS that means
+#     environment.systemPackages or the user's profile -- NOT only inside a
+#     nix-shell, or this will fail at the install step)
+#   - pm2 is already managing NoteServer (pm2 resurrect on boot)
+#   - nginx serves $WEBROOT/current with an SPA fallback to index.html
+#
+# nginx is NOT touched here. On NixOS it is declarative -- see configs/.
+#
+# --- the web root --------------------------------------------------------
+#
+# The built client is published to /var/www/ss, NOT served out of the repo.
+# nginx runs as its own user, so serving from a clone under /home/mab meant
+# loosening the home directory's permissions for every parent dir. /var/www is
+# the conventional place for this and sidesteps the whole problem.
+#
+# Layout, which is the standard symlink-swap pattern:
+#
+#   /var/www/ss/
+#     releases/2026-10-28_1147/   <- one directory per deploy
+#     releases/2026-10-28_0930/
+#     current -> releases/2026-10-28_1147
+#
+# nginx points at the stable path $WEBROOT/current. Publishing is a rename of
+# that symlink, which is a single atomic syscall: no request ever sees a
+# half-copied directory, and no window exists where the root is missing.
+# Rolling back is the same rename pointing at the previous release.
+#
+# Only the *client* moves. The API server still runs from the repo under pm2,
+# and user uploads are still served by Express from $REPO/staticFiles at
+# /api/static -- deliberately, so deploys never touch user data.
+#
+# ONE-TIME SETUP on a new server (needs root):
+#
+#   sudo mkdir -p /var/www/ss/releases
+#   sudo chown -R "$USER" /var/www/ss
+#   sudo chmod 755 /var/www /var/www/ss
+#
+# Override the location for a staging box with:  WEBROOT=/var/www/ss-staging ./deploy.sh
+#
+# --- the database account ------------------------------------------------
+#
+# The API connects as a least-privilege account with DML rights only. This is
+# deliberate: the server fetches untrusted URLs for link previews, and a
+# service able to reshape or drop its own tables is a far larger blast radius
+# than that feature warrants.
+#
+#   CREATE USER 'app'@'localhost' IDENTIFIED BY '<password>';
+#   GRANT SELECT, INSERT, UPDATE, DELETE ON application.* TO 'app'@'localhost';
+#   FLUSH PRIVILEGES;
+#
+#   -- deliberately NOT granted: CREATE, ALTER, DROP, INDEX
+#
+# Then set DB_USER=app and DB_PASS=<password> in ~/.env. Note the asymmetry:
+# the admin scripts here (backupDatabase.sh, applyLastBackup.sh,
+# migrateDatabase.sh) still connect as root with $PROD_DB_PASS.
+#
+# DELETE is genuinely required -- the scraper drops shell attachment rows for
+# dead links and purges expired cache entries.
+#
+# --- schema changes ------------------------------------------------------
+#
+# Schema is managed by Prisma, for migrations and scaffolding ONLY -- the
+# running server never imports it and still talks to MariaDB through raw SQL.
+# The schema lives in prisma/schema.prisma; migrations in prisma/migrations/.
+#
+# Because the app holds no DDL, a deploy that introduces schema changes needs
+# one manual step, run once per environment as a database admin:
+#
+#   nix-shell shell.nix --run ./migrateDatabase.sh
+#
+# nix-shell is required: the prisma CLI comes from nixpkgs (prisma_7, 7.8.0),
+# not from node_modules. Alternatively add prisma_7 to the server's
+# environment.systemPackages and call the script directly.
+#
+# It is idempotent, so re-running is safe. On a database that predates Prisma
+# the script detects the missing migration history and baselines it
+# automatically (marking 0_init applied rather than executing it) -- it does
+# NOT need to be done by hand.
+#
+# The API verifies the schema and the grants at boot; if either is missing it
+# logs the remedy and serves normally with link previews disabled.
+#
+# NEVER run `prisma migrate dev`, `prisma migrate reset` or `prisma db push`.
+# The first wants a shadow database; the latter two can drop data.
+#
+# DATABASE_URL is composed by the script from the admin credentials in ~/.env
+# and is never read by the application.
+#
+# Note: _prisma_migrations lives inside the `application` database, so the
+# least-privilege `app` user can write migration history (its grant is
+# database-wide). It holds no DDL, so it cannot reshape tables -- narrowing
+# this further would mean per-table grants, which is a separate project.
+#
+
+set -euo pipefail
+
+REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$REPO"
+
+WEBROOT="${WEBROOT:-/var/www/ss}"
+RELEASE="$WEBROOT/releases/$(date +%Y-%m-%d_%H%M%S)"
+# Number of old releases to keep around for rollback.
+KEEP_RELEASES=5
+
+step(){ printf '\n\033[32m::--:: %s\033[0m\n' "$1"; }
+
+# --- sanity checks before changing anything ------------------------------
+
+for tool in git pnpm pm2 rsync; do
+	command -v "$tool" >/dev/null || { echo "$tool is not on PATH" >&2; exit 1; }
+done
+
+# Fail here rather than after a three-minute build. A missing web root is
+# almost always a server that never had the one-time setup above run on it.
+if [ ! -d "$WEBROOT/releases" ]; then
+	echo "$WEBROOT/releases does not exist -- run the one-time setup at the top of this script." >&2
+	exit 1
+fi
+
+if [ ! -w "$WEBROOT/releases" ]; then
+	echo "$WEBROOT/releases is not writable by $(whoami) -- check the chown in the one-time setup." >&2
+	exit 1
+fi
+
+if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+	echo "Working tree has local modifications. Commit or stash them first:" >&2
+	git status --short >&2
+	exit 1
+fi
+
+# --- pull ----------------------------------------------------------------
+
+step "Pulling $(git rev-parse --abbrev-ref HEAD)..."
+# --ff-only: refuse to create a merge commit on the server. If this fails the
+# server has diverged and wants a human.
+git pull --ff-only
+
+# --- dependencies --------------------------------------------------------
+
+step "Installing server dependencies..."
+pnpm install --frozen-lockfile
+
+step "Installing client dependencies..."
+pnpm install --frozen-lockfile --dir client
+
+# --- build ---------------------------------------------------------------
+
+# dist is gitignored, so the server builds it. The build stays inside the repo
+# and is only copied to $WEBROOT once it has been checked, so a failed build
+# cannot touch what is currently being served.
+step "Building client..."
+rm -rf client/dist
+pnpm --dir client exec vite build
+
+# manifest.json is checked in under client/public and copied verbatim by Vite;
+# if it is missing, the PWA installs without icons, scope or a share target.
+for required in index.html sw.js manifest.json; do
+	if [ ! -f "client/dist/$required" ]; then
+		echo "Build output looks wrong (missing $required) -- $WEBROOT left untouched." >&2
+		exit 1
+	fi
+done
+
+# --- publish -------------------------------------------------------------
+
+step "Publishing to $RELEASE..."
+mkdir -p "$RELEASE"
+# Trailing slash on the source: copy the *contents* of dist into the release
+# dir. --delete is harmless on a fresh dir but keeps a re-run of the same
+# timestamp honest.
+rsync -a --delete client/dist/ "$RELEASE/"
+
+# nginx reads these as its own user, so they must be world-readable.
+chmod -R a+rX "$RELEASE"
+
+step "Pointing $WEBROOT/current at the new release..."
+# Two steps because `ln -sfn` over an existing symlink unlinks before relinking,
+# which leaves a brief window with no web root at all. Creating a scratch
+# symlink and renaming it over the old one is a single atomic rename(2).
+ln -sfn "$RELEASE" "$WEBROOT/current.new"
+mv -T "$WEBROOT/current.new" "$WEBROOT/current"
+
+# --- prune ---------------------------------------------------------------
+
+# Keep the most recent $KEEP_RELEASES for rollback, drop the rest. Never remove
+# whatever `current` actually resolves to, however old it is.
+live="$(readlink -f "$WEBROOT/current")"
+ls -1d "$WEBROOT"/releases/*/ 2>/dev/null \
+	| sort -r \
+	| tail -n +$((KEEP_RELEASES + 1)) \
+	| while read -r stale; do
+		[ "$(readlink -f "$stale")" = "$live" ] && continue
+		echo "  pruning $(basename "$stale")"
+		rm -rf "$stale"
+	done
+
+# --- restart -------------------------------------------------------------
+
+step "Reloading the API server..."
+# reload, not restart: graceful in cluster mode. --update-env so changes to
+# ecosystem.config.js are picked up.
+pm2 reload "$REPO/server/ecosystem.config.js" --env production --update-env
+
+# Persist the process list so `pm2 resurrect` on boot gets the current state.
+pm2 save
+
+step "Done."
+echo "  serving $(readlink -f "$WEBROOT/current")"
+echo
+echo "  To roll back, repoint the symlink at the previous release:"
+echo "    ls $WEBROOT/releases"
+echo "    ln -sfn $WEBROOT/releases/<older> $WEBROOT/current.new && mv -T $WEBROOT/current.new $WEBROOT/current"
+echo
+pm2 list

jest.config.js

@@ -1,11 +1,19 @@
-const path = '../../'
+//
-const prefix = '/$1'
+// The @alias map must mirror _moduleAliases in package.json, which is what
+// module-alias/register sets up at runtime.
+//
+// These were previously written as "../../server/..." relative paths, which
+// jest resolves relative to the requiring file - so they never actually
+// resolved, and every existing test sidestepped them with relative requires.
+// <rootDir> is depth-independent and works from any test directory.
+//
 module.exports = {
    moduleNameMapper: {
-    "@root/(.*)": ".",
+    "^@root/(.*)$": "<rootDir>/$1",
-    "@models/(.*)": path+"server/models"+prefix,
+    "^@models/(.*)$": "<rootDir>/server/models/$1",
-    "@routes/(.*)": path+"server/routes"+prefix,
+    "^@routes/(.*)$": "<rootDir>/server/routes/$1",
-    "@helpers/(.*)": path+"server/helpers"+prefix,
+    "^@helpers/(.*)$": "<rootDir>/server/helpers/$1",
-    "@config/(.*)": path+"server/config"+prefix,
+    "^@config/(.*)$": "<rootDir>/server/config/$1",
+    "^@workers/(.*)$": "<rootDir>/server/workers/$1",
    }
 }

package.json

@@ -10,7 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "body-parser": "^1.19.0",
-    "cheerio": "^1.0.0-rc.3",
+    "cheerio": "^1.1.0",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
     "express-rate-limit": "^5.1.3",
@@ -22,8 +22,6 @@
     "mysql2": "^2.2.5",
     "node-tesseract-ocr": "^2.0.0",
     "qrcode": "^1.4.4",
-    "request": "^2.88.2",
-    "request-promise": "^4.2.6",
     "socket.io": "^2.3.0",
     "speakeasy": "^2.0.0"
   },
@@ -32,7 +30,8 @@
     "@models": "server/models",
     "@routes": "server/routes",
     "@helpers": "server/helpers",
-    "@config": "server/config"
+    "@config": "server/config",
+    "@workers": "server/workers"
   },
   "devDependencies": {
     "jest": "^29.7.0"

server/config/scrape.js

@@ -0,0 +1,70 @@
+//
+// All scraper tunables in one place, so nothing downstream is a magic number.
+//
+// Env overrides exist for the two kill switches only. Everything else is a
+// constant on purpose: the politeness numbers are the difference between
+// looking like a browser and looking like a bot, and they should not drift
+// per-environment without a code change.
+//
+
+const isTest = process.env.NODE_ENV === 'test'
+
+const config = {
+	// Kill switches
+	enabled: process.env.SCRAPE_ENABLED !== '0',
+	// The background worker is off under Jest so tests drive the queue
+	// deterministically via ScrapeQueue.drain() instead of racing it.
+	workerEnabled: process.env.SCRAPE_WORKER ? process.env.SCRAPE_WORKER === '1' : !isTest,
+
+	// Timeouts
+	pageTimeoutMs: 8000,     // single HTTP request
+	totalBudgetMs: 25000,    // all hops + retries for one URL
+	imageTimeoutMs: 10000,
+
+	// Size caps. Content-Length is optional and can lie, so these are also
+	// enforced while streaming the body.
+	maxHtmlBytes: 2 * 1024 * 1024,
+	maxImageBytes: 5 * 1024 * 1024,
+	maxRobotsBytes: 100 * 1024,
+
+	// Redirects
+	maxRedirects: 5,
+
+	// Retries. Only for transient classes - a 404 is an answer, not a failure.
+	retries: 2,
+	backoffBaseMs: 500,
+	backoffMaxMs: 8000,
+	maxRetryAfterMs: 60 * 1000,
+
+	// Politeness. One request at a time per host, spaced out, is what actually
+	// stops us looking like a crawler.
+	maxConcurrentFetches: 4,
+	perHostConcurrency: 1,
+	perHostMinSpacingMs: 1000,
+	perHostMaxSpacingMs: 3000,
+	maxCrawlDelayMs: 30 * 1000,
+
+	// Backpressure
+	maxQueueLength: 500,
+	maxPendingPerUser: 50,
+
+	// Identity. Stable salt, so a host always sees the same browser across
+	// restarts. Changing this re-rolls every host's assigned profile.
+	profileSalt: 'ss-scrape-v1',
+	robotsUaToken: 'SolidScribeBot',
+	robotsTtlMs: 24 * 60 * 60 * 1000,
+	robotsNegativeTtlMs: 60 * 60 * 1000,
+	robotsErrorTtlMs: 10 * 60 * 1000,
+	robotsMaxOrigins: 2000,
+
+	// Parsing / storage
+	maxSearchTextBytes: 8000,
+	maxKeywordSourceChars: 6000,
+	maxUrlLength: 1000, // matches attachment.url varchar(1000)
+
+	// staticFiles is served from our own origin via /api/static, so an SVG
+	// with an inline script would be same-origin. Off by default.
+	allowSvgThumbnails: false,
+}
+
+module.exports = config

server/ecosystem.config.js

@@ -2,12 +2,19 @@ module.exports = {
   apps : [{
     name: 'NoteServer',
     script: 'index.js',
+    // Both relative to this file, not to whatever directory pm2 was invoked
+    // from. deploy.sh runs pm2 from the repo root, so without cwd pm2 resolves
+    // script against the root and dies with "Script not found: .../index.js".
+    cwd: __dirname,
 
     // Options reference: https://pm2.io/doc/en/runtime/reference/ecosystem-file/
     // args: 'one two',
     instances: 1,
     autorestart: true,
-    watch: true,
+    // Off by default so it is safe in production: with watch on, a `git pull`
+    // that touches server files restarts pm2 mid-pull, potentially booting a
+    // half-updated tree. startLocalDev.sh passes --watch for development.
+    watch: false,
     ignore_watch : ["node_modules", "staticFiles"],
     max_memory_restart: '1G',
     env: {
@@ -18,14 +25,4 @@ module.exports = {
     }
   }],
 
-  deploy : {
-    production : {
-      user : 'node',
-      host : '212.83.163.1',
-      ref  : 'origin/master',
-      repo : '[email protected]:repo.git',
-      path : '/var/www/production',
-      'post-deploy' : 'npm install && pm2 reload ecosystem.config.js --env production'
-    }
-  }
 };

server/helpers/BrowserProfiles.js

@@ -0,0 +1,195 @@
+//
+// Browser identities for outbound fetches.
+//
+// The old code sent a single hardcoded Googlebot User-Agent. That was removed
+// deliberately: Googlebot is verifiable by reverse DNS, so the claim is trivially
+// disproven, and sites that check it block the source IP outright. Some also
+// serve crawlers a stripped-down page.
+//
+// What replaces it is an honest, *coherent* browser identity. Coherence is the
+// point - a Chrome User-Agent shipping Firefox's Accept header, or sec-ch-ua on
+// a Safari request, is a stronger bot signal than any UA string. So each profile
+// carries the complete header set its family really sends, and nothing mixes.
+//
+// Profiles are assigned stably per host, not randomly per request: a site should
+// see one returning browser, not a rotating cast.
+//
+
+const config = require('@config/scrape')
+
+let BrowserProfiles = module.exports = {}
+
+const CHROME_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'
+const FIREFOX_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
+const SAFARI_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+
+const PROFILES = BrowserProfiles.PROFILES = [
+	{
+		id: 'chrome-win',
+		family: 'chrome',
+		ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+		accept: CHROME_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.9',
+		secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
+		secChUaMobile: '?0',
+		secChUaPlatform: '"Windows"',
+	},
+	{
+		id: 'chrome-mac',
+		family: 'chrome',
+		ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+		accept: CHROME_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.9',
+		secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
+		secChUaMobile: '?0',
+		secChUaPlatform: '"macOS"',
+	},
+	{
+		id: 'firefox-win',
+		family: 'firefox',
+		ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
+		accept: FIREFOX_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.5',
+	},
+	{
+		id: 'firefox-linux',
+		family: 'firefox',
+		ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0',
+		accept: FIREFOX_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.5',
+	},
+	{
+		id: 'safari-mac',
+		family: 'safari',
+		ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
+		accept: SAFARI_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.9',
+	},
+]
+
+// FNV-1a. Not cryptographic - we only need a stable, well-spread bucket.
+const fnv1a = (input) => {
+	let hash = 0x811c9dc5
+	for (let i = 0; i < input.length; i++) {
+		hash ^= input.charCodeAt(i)
+		hash = Math.imul(hash, 0x01000193) >>> 0
+	}
+	return hash >>> 0
+}
+BrowserProfiles.__fnv1a = fnv1a
+
+//
+// Same host always gets the same profile, across requests and across restarts
+// (the salt is a constant in config, not a per-boot random).
+//
+BrowserProfiles.profileForHost = (hostname) => {
+	const key = require('@helpers/UrlSafety').registrableHost(hostname) + config.profileSalt
+	return PROFILES[fnv1a(key) % PROFILES.length]
+}
+
+BrowserProfiles.profileById = (id) => PROFILES.find(profile => profile.id === id) || PROFILES[0]
+
+//
+// Sec-Fetch-Site is relative: "none" for a URL typed into the bar, "same-origin"
+// when the referrer shares the origin, "cross-site" otherwise. Getting this
+// wrong while sending a Referer is exactly the kind of incoherence that flags.
+//
+const fetchSite = (targetUrl, referer) => {
+	if (!referer) { return 'none' }
+	try {
+		const target = new URL(targetUrl)
+		const from = new URL(referer)
+		if (target.origin === from.origin) { return 'same-origin' }
+		if (UrlSafetyHost(target) === UrlSafetyHost(from)) { return 'same-site' }
+		return 'cross-site'
+	} catch (error) {
+		return 'none'
+	}
+}
+
+const UrlSafetyHost = (url) => require('@helpers/UrlSafety').registrableHost(url.hostname)
+
+const applyChromiumHints = (headers, profile) => {
+	if (profile.family !== 'chrome') { return headers }
+	headers['sec-ch-ua'] = profile.secChUa
+	headers['sec-ch-ua-mobile'] = profile.secChUaMobile
+	headers['sec-ch-ua-platform'] = profile.secChUaPlatform
+	return headers
+}
+
+//
+// Headers for a top-level document navigation.
+//
+BrowserProfiles.documentHeaders = (profile, { url, referer } = {}) => {
+	const headers = {
+		'User-Agent': profile.ua,
+		'Accept': profile.accept,
+		'Accept-Language': profile.acceptLanguage,
+		'Accept-Encoding': 'gzip, deflate, br',
+	}
+
+	// Safari sends neither sec-ch-ua nor Sec-Fetch-*; Firefox sends Sec-Fetch-*
+	// but no client hints; Chromium sends both.
+	if (profile.family !== 'safari') {
+		headers['Sec-Fetch-Dest'] = 'document'
+		headers['Sec-Fetch-Mode'] = 'navigate'
+		headers['Sec-Fetch-Site'] = fetchSite(url, referer)
+		headers['Sec-Fetch-User'] = '?1'
+		headers['Upgrade-Insecure-Requests'] = '1'
+	}
+
+	applyChromiumHints(headers, profile)
+
+	if (referer) { headers['Referer'] = referer }
+
+	return headers
+}
+
+//
+// Headers for a subresource image load, as issued from the page it was found on.
+//
+BrowserProfiles.imageHeaders = (profile, { url, pageUrl } = {}) => {
+	const headers = {
+		'User-Agent': profile.ua,
+		'Accept': profile.family === 'chrome'
+			? 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8'
+			: 'image/avif,image/webp,*/*',
+		'Accept-Language': profile.acceptLanguage,
+		'Accept-Encoding': 'gzip, deflate, br',
+	}
+
+	if (profile.family !== 'safari') {
+		headers['Sec-Fetch-Dest'] = 'image'
+		headers['Sec-Fetch-Mode'] = 'no-cors'
+		headers['Sec-Fetch-Site'] = fetchSite(url, pageUrl)
+	}
+
+	applyChromiumHints(headers, profile)
+
+	if (pageUrl) { headers['Referer'] = pageUrl }
+
+	return headers
+}
+
+//
+// Headers for robots.txt. Same identity as the page fetch, but it is not a
+// navigation - a browser would never send Sec-Fetch-Dest: document for this.
+//
+BrowserProfiles.robotsHeaders = (profile) => {
+	const headers = {
+		'User-Agent': profile.ua,
+		'Accept': 'text/plain,*/*;q=0.8',
+		'Accept-Language': profile.acceptLanguage,
+		'Accept-Encoding': 'gzip, deflate, br',
+	}
+
+	if (profile.family !== 'safari') {
+		headers['Sec-Fetch-Dest'] = 'empty'
+		headers['Sec-Fetch-Mode'] = 'no-cors'
+		headers['Sec-Fetch-Site'] = 'none'
+	}
+
+	applyChromiumHints(headers, profile)
+
+	return headers
+}

server/helpers/HostQueue.js

@@ -0,0 +1,221 @@
+//
+// Per-host politeness scheduler. Knows nothing about scraping.
+//
+// This is the piece that stops the scraper looking like a crawler. The old code
+// did foundUrls.forEach(fetch) - a note with 40 links opened 40 simultaneous
+// connections, which is the most bot-like traffic shape available. Here, each
+// host gets one request at a time with jittered spacing between them, and a
+// global cap bounds total concurrency.
+//
+
+const config = require('@config/scrape')
+
+let HostQueue = module.exports = {}
+
+const jitter = (min, max) => Math.round(min + Math.random() * Math.max(0, max - min))
+
+class Queue {
+	constructor(options = {}) {
+		this.globalConcurrency = options.globalConcurrency != null
+			? options.globalConcurrency : config.maxConcurrentFetches
+		this.perHostConcurrency = options.perHostConcurrency != null
+			? options.perHostConcurrency : config.perHostConcurrency
+		this.minSpacingMs = options.minSpacingMs != null
+			? options.minSpacingMs : config.perHostMinSpacingMs
+		this.maxSpacingMs = options.maxSpacingMs != null
+			? options.maxSpacingMs : config.perHostMaxSpacingMs
+		this.now = options.now || (() => Date.now())
+		// Injectable so tests with a controlled clock don't spend real
+		// wall-time waiting out politeness spacing.
+		this.setTimeoutFn = options.setTimeoutFn || ((fn, ms) => setTimeout(fn, ms))
+		this.clearTimeoutFn = options.clearTimeoutFn || ((handle) => clearTimeout(handle))
+
+		this.pending = []            // FIFO
+		this.hosts = new Map()       // hostKey -> { running, nextAllowedAt }
+		this.running = 0
+		this.seq = 0
+		this.timer = null
+		this.stopped = false
+		this.drainWaiters = []
+	}
+
+	hostState(hostKey) {
+		if (!this.hosts.has(hostKey)) {
+			this.hosts.set(hostKey, { running: 0, nextAllowedAt: 0 })
+		}
+		return this.hosts.get(hostKey)
+	}
+
+	//
+	// Queue a task for a host. Resolves/rejects with the task's own outcome.
+	// spacingMs overrides the jitter floor (robots.txt Crawl-delay).
+	//
+	schedule(hostKey, taskFn, options = {}) {
+		if (this.stopped) {
+			return Promise.reject(new Error('HostQueue stopped'))
+		}
+
+		return new Promise((resolve, reject) => {
+			this.pending.push({
+				hostKey: hostKey || '(none)',
+				taskFn,
+				resolve,
+				reject,
+				spacingMs: options.spacingMs != null ? options.spacingMs : null,
+				seq: this.seq++,
+			})
+			this.pump()
+		})
+	}
+
+	//
+	// Run whatever is eligible. FIFO scan rather than per-host round-robin, so
+	// no host can starve another and order stays predictable.
+	//
+	pump() {
+		if (this.stopped) { return }
+
+		this.clearTimer()
+
+		while (this.running < this.globalConcurrency) {
+			const at = this.now()
+			let index = -1
+			let earliest = Infinity
+
+			for (let i = 0; i < this.pending.length; i++) {
+				const job = this.pending[i]
+				const host = this.hostState(job.hostKey)
+
+				if (host.running >= this.perHostConcurrency) { continue }
+
+				if (host.nextAllowedAt > at) {
+					earliest = Math.min(earliest, host.nextAllowedAt)
+					continue
+				}
+
+				index = i
+				break
+			}
+
+			if (index === -1) {
+				// Nothing runnable now. Wake up when the earliest host is due.
+				if (this.pending.length && earliest !== Infinity) {
+					this.armTimer(earliest - at)
+				}
+				break
+			}
+
+			this.run(this.pending.splice(index, 1)[0])
+		}
+
+		this.checkDrained()
+	}
+
+	run(job) {
+		const host = this.hostState(job.hostKey)
+		host.running++
+		this.running++
+
+		const settle = () => {
+			host.running--
+			this.running--
+
+			// Space the NEXT request to this host from the end of this one.
+			const spacing = job.spacingMs != null
+				? job.spacingMs
+				: jitter(this.minSpacingMs, this.maxSpacingMs)
+			host.nextAllowedAt = this.now() + spacing
+
+			this.pump()
+		}
+
+		// A throwing taskFn must not wedge the pump, so the call itself is
+		// wrapped rather than only its promise.
+		let result = null
+		try {
+			result = Promise.resolve(job.taskFn())
+		} catch (error) {
+			result = Promise.reject(error)
+		}
+
+		result.then(
+			value => { settle(); job.resolve(value) },
+			error => { settle(); job.reject(error) }
+		)
+	}
+
+	armTimer(delay) {
+		this.clearTimer()
+		this.timer = this.setTimeoutFn(() => {
+			this.timer = null
+			this.pump()
+		}, Math.max(1, delay))
+
+		//
+		// Deliberately NOT unref'd.
+		//
+		// This timer is the only thing that will resume a host which is inside
+		// its politeness gap. unref'ing it means that if nothing else holds the
+		// event loop open, Node exits and every queued job is silently
+		// abandoned with its promise never settling. Inside the server that is
+		// masked by the listening socket, but it makes the queue unusable from
+		// a standalone script and is a footgun either way.
+		//
+		// Shutdown is handled explicitly instead: stop() clears this timer and
+		// rejects whatever is still queued, and it is wired to SIGINT/SIGTERM
+		// in server/index.js.
+		//
+	}
+
+	clearTimer() {
+		if (this.timer) {
+			this.clearTimeoutFn(this.timer)
+			this.timer = null
+		}
+	}
+
+	size() { return this.pending.length }
+	inflight() { return this.running }
+
+	stats() {
+		return {
+			pending: this.pending.length,
+			running: this.running,
+			hosts: this.hosts.size,
+		}
+	}
+
+	checkDrained() {
+		if (this.pending.length || this.running) { return }
+
+		const waiters = this.drainWaiters
+		this.drainWaiters = []
+		waiters.forEach(resolve => resolve())
+	}
+
+	// Resolves once the queue is empty and nothing is in flight.
+	drain() {
+		if (!this.pending.length && !this.running) { return Promise.resolve() }
+		return new Promise(resolve => this.drainWaiters.push(resolve))
+	}
+
+	//
+	// Reject everything still queued. In-flight tasks are not cancelled here -
+	// their own AbortControllers handle that.
+	//
+	stop() {
+		this.stopped = true
+		this.clearTimer()
+
+		const pending = this.pending
+		this.pending = []
+		pending.forEach(job => job.reject(new Error('HostQueue stopped')))
+
+		this.checkDrained()
+	}
+}
+
+HostQueue.Queue = Queue
+
+// Shared instance used by the scrape worker. Tests build their own.
+HostQueue.shared = new Queue()

server/helpers/HttpFetch.js

@@ -0,0 +1,429 @@
+//
+// The only module in the server that opens an outbound connection.
+//
+// Everything that made the old scraper unreliable is handled here, in one place:
+// a single AbortController-based timeout (the old code raced request's socket
+// timeout against a manual setTimeout that could fire against a null reference
+// and crash the process), manual redirect following so the SSRF gate runs on
+// every hop, hard size caps enforced while streaming, a Content-Type gate before
+// any body is read, real charset decoding, and retries that only fire for
+// transient failures.
+//
+
+const config = require('@config/scrape')
+const UrlSafety = require('@helpers/UrlSafety')
+const BrowserProfiles = require('@helpers/BrowserProfiles')
+
+let HttpFetch = module.exports = {}
+
+const HTML_TYPES = ['text/html', 'application/xhtml+xml', 'text/plain', 'application/xml', 'text/xml']
+
+class FetchError extends Error {
+	constructor(kind, message, extra = {}) {
+		super(message)
+		this.name = 'FetchError'
+		this.kind = kind          // timeout|http|blocked|too_large|bad_content_type|redirect_loop|too_many_redirects|network|aborted
+		this.status = extra.status || null
+		this.retryAfterMs = extra.retryAfterMs || null
+		this.url = extra.url || null
+		this.reason = extra.reason || null
+	}
+}
+HttpFetch.FetchError = FetchError
+
+// Injectable so tests don't spend real wall-time in backoff. Far simpler than
+// interleaving fake timers with promises.
+let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+HttpFetch.__setSleep = (fn) => { sleep = fn }
+HttpFetch.__resetSleep = () => { sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) }
+
+const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504, 507, 509, 521, 522, 523, 524])
+
+const isRetryable = (error) => {
+	if (!(error instanceof FetchError)) { return false }
+	if (error.kind === 'network' || error.kind === 'timeout') { return true }
+	if (error.kind === 'http') { return RETRYABLE_STATUS.has(error.status) }
+	return false
+}
+
+// Retry-After is either delta-seconds or an HTTP-date.
+const parseRetryAfter = (value) => {
+	if (!value) { return null }
+
+	const seconds = Number(value)
+	if (Number.isFinite(seconds)) { return Math.max(0, seconds * 1000) }
+
+	const when = Date.parse(value)
+	if (Number.isFinite(when)) { return Math.max(0, when - Date.now()) }
+
+	return null
+}
+
+const backoffFor = (attempt) => {
+	const base = Math.min(config.backoffMaxMs, config.backoffBaseMs * Math.pow(2, attempt))
+	return Math.round(base * (0.5 + Math.random())) // +/-50% jitter
+}
+
+const contentTypeOf = (response) => String(response.headers.get('content-type') || '').toLowerCase()
+const mimeOf = (contentType) => contentType.split(';')[0].trim()
+
+const charsetFromContentType = (contentType) => {
+	const match = contentType.match(/charset=\s*"?([^";\s]+)"?/i)
+	return match ? match[1].toLowerCase() : null
+}
+
+// Last resort: sniff a <meta charset> out of the head, read as latin1 so every
+// byte maps to a character and the regex can't miss.
+const charsetFromMeta = (buffer) => {
+	const head = buffer.subarray(0, 2048).toString('latin1')
+
+	const metaCharset = head.match(/<meta[^>]+charset\s*=\s*["']?\s*([a-z0-9_\-:]+)/i)
+	if (metaCharset) { return metaCharset[1].toLowerCase() }
+
+	const httpEquiv = head.match(/<meta[^>]+content\s*=\s*["'][^"']*charset=\s*([a-z0-9_\-:]+)/i)
+	if (httpEquiv) { return httpEquiv[1].toLowerCase() }
+
+	return null
+}
+
+const charsetFromBom = (buffer) => {
+	if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) { return 'utf-8' }
+	if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { return 'utf-16le' }
+	if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) { return 'utf-16be' }
+	return null
+}
+
+const decodeBody = (buffer, declaredCharset) => {
+	const label = declaredCharset || charsetFromBom(buffer) || charsetFromMeta(buffer) || 'utf-8'
+
+	try {
+		return { text: new TextDecoder(label, { fatal: false }).decode(buffer), charset: label }
+	} catch (error) {
+		// Unknown/bogus label - utf-8 with replacement chars beats throwing.
+		return { text: new TextDecoder('utf-8', { fatal: false }).decode(buffer), charset: 'utf-8' }
+	}
+}
+
+//
+// Read the body with a running byte count, aborting mid-stream once the cap is
+// passed. Content-Length is checked by the caller too, but it is optional and
+// can lie, so this is the cap that actually holds.
+//
+const readCapped = async (response, maxBytes, controller, url) => {
+	if (!response.body) { return Buffer.alloc(0) }
+
+	const chunks = []
+	let total = 0
+
+	for await (const chunk of response.body) {
+		const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+		total += buffer.length
+
+		if (total > maxBytes) {
+			try { controller.abort() } catch (error) { /* already done */ }
+			throw new FetchError('too_large', `Body exceeded ${maxBytes} bytes`, { url })
+		}
+
+		chunks.push(buffer)
+	}
+
+	return Buffer.concat(chunks, total)
+}
+
+//
+// One HTTP request, following redirects by hand.
+//
+// Manual redirects are the whole point: fetch's own 'follow' would hop to
+// 127.0.0.1 without ever re-consulting the safety gate.
+//
+const requestOnce = async (startUrl, options) => {
+	const {
+		maxBytes,
+		acceptMimes,
+		headersFor,
+		conditional,
+		timeoutMs,
+		deadline,
+		referer,
+	} = options
+
+	let currentUrl = typeof startUrl === 'string' ? startUrl : startUrl.href
+	const visited = new Set()
+
+	for (let hop = 0; hop <= config.maxRedirects; hop++) {
+		if (visited.has(currentUrl)) {
+			throw new FetchError('redirect_loop', `Redirect loop at ${currentUrl}`, { url: currentUrl })
+		}
+		visited.add(currentUrl)
+
+		// Re-validate on every hop. This is the check the old code never had.
+		try {
+			await UrlSafety.assertFetchable(currentUrl)
+		} catch (error) {
+			// A DNS failure is transient - the host may simply be down right
+			// now - so it must not be classified alongside "this address is
+			// private", which is a permanent refusal. Collapsing the two meant
+			// a briefly-unresolvable domain got parked for the full cache TTL
+			// and had its attachment row deleted.
+			const kind = error.reason === 'dns_failed' ? 'network' : 'blocked'
+			throw new FetchError(kind, error.message, { url: currentUrl, reason: error.reason })
+		}
+
+		const remaining = deadline - Date.now()
+		if (remaining <= 0) {
+			throw new FetchError('timeout', 'Total budget exhausted', { url: currentUrl })
+		}
+
+		const controller = new AbortController()
+		const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, remaining))
+
+		let response = null
+		try {
+			const headers = headersFor(currentUrl, referer)
+
+			if (conditional && conditional.etag) { headers['If-None-Match'] = conditional.etag }
+			if (conditional && conditional.lastModified) { headers['If-Modified-Since'] = conditional.lastModified }
+
+			response = await fetch(currentUrl, {
+				method: 'GET',
+				redirect: 'manual',
+				signal: controller.signal,
+				headers,
+			})
+		} catch (error) {
+			clearTimeout(timer)
+
+			if (controller.signal.aborted) {
+				throw new FetchError('timeout', `Timed out after ${timeoutMs}ms`, { url: currentUrl })
+			}
+			throw new FetchError('network', error.message || 'Network failure', { url: currentUrl })
+		}
+
+		// Redirect?
+		if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
+			clearTimeout(timer)
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* nothing to drain */ }
+
+			const location = response.headers.get('location')
+			let next = null
+			try {
+				next = new URL(location, currentUrl)
+			} catch (error) {
+				throw new FetchError('network', `Unparseable redirect target ${location}`, { url: currentUrl })
+			}
+
+			// Never follow https -> http. A downgrade mid-chain is either an
+			// attack or a misconfiguration, and is not worth the preview.
+			if (new URL(currentUrl).protocol === 'https:' && next.protocol === 'http:') {
+				throw new FetchError('blocked', `Refusing https->http downgrade to ${next.href}`, { url: currentUrl })
+			}
+
+			currentUrl = next.href
+			continue
+		}
+
+		clearTimeout(timer)
+
+		if (response.status === 304) {
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
+			return { notModified: true, status: 304, finalUrl: currentUrl, response }
+		}
+
+		if (!response.ok) {
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
+			throw new FetchError('http', `HTTP ${response.status} for ${currentUrl}`, {
+				status: response.status,
+				retryAfterMs: parseRetryAfter(response.headers.get('retry-after')),
+				url: currentUrl,
+			})
+		}
+
+		const contentType = contentTypeOf(response)
+		const mime = mimeOf(contentType)
+
+		// Gate on type BEFORE reading a byte. This is what stops a 100MB binary
+		// being handed to cheerio.
+		if (acceptMimes && !acceptMimes(mime)) {
+			try { controller.abort() } catch (error) { /* ignore */ }
+			throw new FetchError('bad_content_type', `Refusing content-type ${mime || '(none)'}`, { url: currentUrl })
+		}
+
+		const declaredLength = Number(response.headers.get('content-length'))
+		if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
+			try { controller.abort() } catch (error) { /* ignore */ }
+			throw new FetchError('too_large', `Declared length ${declaredLength} over cap`, { url: currentUrl })
+		}
+
+		const buffer = await readCapped(response, maxBytes, controller, currentUrl)
+
+		return {
+			notModified: false,
+			status: response.status,
+			finalUrl: currentUrl,
+			contentType,
+			mime,
+			buffer,
+			response,
+		}
+	}
+
+	throw new FetchError('too_many_redirects', `More than ${config.maxRedirects} redirects`, { url: currentUrl })
+}
+
+//
+// Retry wrapper. Only transient classes are retried - a 404 is an answer.
+//
+const withRetries = async (attemptFn, { retries, deadline }) => {
+	let lastError = null
+
+	for (let attempt = 0; attempt <= retries; attempt++) {
+		try {
+			return await attemptFn()
+		} catch (error) {
+			lastError = error
+
+			if (!isRetryable(error) || attempt === retries) { throw error }
+
+			// A server asking us to come back in an hour is not something to
+			// sit on - surface it so the cache can park the URL instead.
+			if (error.retryAfterMs && error.retryAfterMs > config.maxRetryAfterMs) { throw error }
+
+			const wait = error.retryAfterMs != null
+				? Math.min(error.retryAfterMs, config.maxRetryAfterMs)
+				: backoffFor(attempt)
+
+			if (Date.now() + wait >= deadline) { throw error }
+
+			await sleep(wait)
+		}
+	}
+
+	throw lastError
+}
+
+//
+// Fetch an HTML document.
+//
+HttpFetch.fetchDocument = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || config.pageTimeoutMs
+	const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
+	const maxBytes = options.maxBytes || config.maxHtmlBytes
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const result = await withRetries(() => requestOnce(url, {
+		maxBytes,
+		acceptMimes: (mime) => HTML_TYPES.includes(mime),
+		headersFor: (target, referer) => BrowserProfiles.documentHeaders(profile, {
+			url: target,
+			// A browser arriving at a page has *some* referer; its own origin is
+			// the least surprising one and pairs with Sec-Fetch-Site: same-origin.
+			referer: referer || originOf(target),
+		}),
+		conditional: options.conditional,
+		timeoutMs,
+		deadline,
+		referer: options.referer,
+	}), { retries: options.retries != null ? options.retries : config.retries, deadline })
+
+	if (result.notModified) {
+		return {
+			notModified: true,
+			status: 304,
+			finalUrl: result.finalUrl,
+			html: null,
+			profileId: profile.id,
+		}
+	}
+
+	const declaredCharset = charsetFromContentType(result.contentType)
+	const { text, charset } = decodeBody(result.buffer, declaredCharset)
+
+	return {
+		notModified: false,
+		status: result.status,
+		finalUrl: result.finalUrl,
+		html: text,
+		charset,
+		contentType: result.contentType,
+		bytes: result.buffer.length,
+		etag: result.response.headers.get('etag') || null,
+		lastModified: result.response.headers.get('last-modified') || null,
+		profileId: profile.id,
+	}
+}
+
+//
+// Fetch a binary (thumbnail image).
+//
+HttpFetch.fetchBinary = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || config.imageTimeoutMs
+	const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
+	const maxBytes = options.maxBytes || config.maxImageBytes
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const acceptMimes = (mime) => {
+		if (!mime.startsWith('image/')) { return false }
+		if (mime === 'image/svg+xml' && !config.allowSvgThumbnails) { return false }
+		return true
+	}
+
+	const result = await withRetries(() => requestOnce(url, {
+		maxBytes,
+		acceptMimes,
+		headersFor: (target) => BrowserProfiles.imageHeaders(profile, {
+			url: target,
+			pageUrl: options.pageUrl,
+		}),
+		timeoutMs,
+		deadline,
+	}), { retries: options.retries != null ? options.retries : config.retries, deadline })
+
+	return {
+		status: result.status,
+		finalUrl: result.finalUrl,
+		contentType: result.contentType,
+		mime: result.mime,
+		buffer: result.buffer,
+		profileId: profile.id,
+	}
+}
+
+//
+// Fetch robots.txt. Short leash, no retries - fail-open is handled by the caller.
+//
+HttpFetch.fetchText = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || 5000
+	const deadline = Date.now() + (options.budgetMs || timeoutMs + 1000)
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const result = await requestOnce(url, {
+		maxBytes: options.maxBytes || config.maxRobotsBytes,
+		acceptMimes: null, // robots.txt is served as every type under the sun
+		headersFor: () => BrowserProfiles.robotsHeaders(profile),
+		timeoutMs,
+		deadline,
+	})
+
+	if (result.notModified) { return { status: 304, text: null, finalUrl: result.finalUrl } }
+
+	const { text } = decodeBody(result.buffer, charsetFromContentType(result.contentType))
+	return { status: result.status, text, finalUrl: result.finalUrl }
+}
+
+const hostnameOf = (url) => {
+	try { return new URL(typeof url === 'string' ? url : url.href).hostname }
+	catch (error) { return '' }
+}
+
+const originOf = (url) => {
+	try { return new URL(url).origin + '/' }
+	catch (error) { return undefined }
+}
+
+//
+// Deliberately NOT attempted here, so nobody wonders later:
+//   - Header ordering: fetch normalises it; controlling order needs raw sockets.
+//   - HTTP/2: undici's fetch is h1 unless an Agent with allowH2 is supplied.
+//   - TLS/JA3 fingerprinting and proxy rotation: ruled out, we are being polite
+//     rather than evasive.
+//

server/helpers/Realtime.js

@@ -0,0 +1,43 @@
+//
+// Socket.IO emit wrapper.
+//
+// Replaces the `if(typeof SocketIo != 'undefined')` guards that were scattered
+// through the models. Those guards did not just avoid a crash - they silently
+// disabled ALL scraping under Jest, which is why none of this code was ever
+// testable. Here the no-socket case is a no-op rather than a skipped branch, so
+// the surrounding logic runs identically in tests and in production.
+//
+
+let Realtime = module.exports = {}
+
+let warned = false
+
+const server = () => (typeof global.SocketIo !== 'undefined' && global.SocketIo) ? global.SocketIo : null
+
+Realtime.isEnabled = () => server() !== null
+
+const emit = (room, event, payload) => {
+	const io = server()
+
+	if (!io) {
+		if (!warned) {
+			warned = true
+			console.log('[realtime] no socket server bound; emits are no-ops')
+		}
+		return false
+	}
+
+	try {
+		if (payload === undefined) { io.to(room).emit(event) }
+		else { io.to(room).emit(event, payload) }
+		return true
+	} catch (error) {
+		console.log('[realtime] emit failed', event, error.message)
+		return false
+	}
+}
+
+Realtime.emitToUser = (userId, event, payload) => emit(`user:${userId}`, event, payload)
+Realtime.emitToNote = (noteRawTextId, event, payload) => emit(`note:${noteRawTextId}`, event, payload)
+
+Realtime.__resetWarning = () => { warned = false }

server/helpers/Robots.js

@@ -0,0 +1,216 @@
+//
+// robots.txt fetching, parsing and caching.
+//
+// Posture: polite, not evasive. An explicit Disallow is respected (fail-closed);
+// a robots.txt we cannot retrieve is treated as permissive (fail-open), because
+// a 404 or a flaky server is not a refusal and these are single-page fetches of
+// URLs a user explicitly saved.
+//
+
+const config = require('@config/scrape')
+const HttpFetch = require('@helpers/HttpFetch')
+
+let Robots = module.exports = {}
+
+// origin -> { rules, expiresAt }
+const cache = new Map()
+
+Robots.__resetCache = () => cache.clear()
+Robots.__cacheSize = () => cache.size
+
+const ALLOW_ALL = { groups: [], allowAll: true }
+
+//
+// Parse robots.txt into user-agent groups.
+//
+// Consecutive User-agent lines share one rule block, which is the part naive
+// parsers get wrong.
+//
+Robots.parse = (text) => {
+	if (!text || typeof text !== 'string') { return ALLOW_ALL }
+
+	const groups = []
+	let current = null
+	let expectingAgents = false
+
+	for (const rawLine of text.split(/\r?\n/)) {
+		const line = rawLine.replace(/#.*$/, '').trim()
+		if (!line) { continue }
+
+		const separator = line.indexOf(':')
+		if (separator === -1) { continue }
+
+		const field = line.slice(0, separator).trim().toLowerCase()
+		const value = line.slice(separator + 1).trim()
+
+		if (field === 'user-agent') {
+			// A User-agent line directly after another starts a shared group.
+			if (!current || !expectingAgents) {
+				current = { agents: [], rules: [], crawlDelayMs: null }
+				groups.push(current)
+			}
+			current.agents.push(value.toLowerCase())
+			expectingAgents = true
+			continue
+		}
+
+		if (!current) { continue } // directive before any user-agent
+		expectingAgents = false
+
+		if (field === 'disallow' || field === 'allow') {
+			// "Disallow:" with an empty value means allow everything.
+			if (field === 'disallow' && value === '') { continue }
+			current.rules.push({ allow: field === 'allow', path: value })
+			continue
+		}
+
+		if (field === 'crawl-delay') {
+			const seconds = Number(value)
+			if (Number.isFinite(seconds) && seconds >= 0) {
+				current.crawlDelayMs = Math.round(seconds * 1000)
+			}
+		}
+	}
+
+	return { groups, allowAll: groups.length === 0 }
+}
+
+//
+// Pick the group whose agent token best matches us. A named match always beats
+// the "*" wildcard, regardless of file order.
+//
+const groupFor = (rules, userAgentToken) => {
+	if (!rules || rules.allowAll) { return null }
+
+	const token = userAgentToken.toLowerCase()
+	let best = null
+	let bestScore = -1
+
+	for (const group of rules.groups) {
+		for (const agent of group.agents) {
+			let score = -1
+			if (agent === '*') { score = 0 }
+			else if (token.includes(agent)) { score = agent.length }
+
+			if (score > bestScore) {
+				bestScore = score
+				best = group
+			}
+		}
+	}
+
+	return best
+}
+
+//
+// Match a robots path pattern, supporting the * and $ wildcards.
+//
+const matchesPattern = (pattern, path) => {
+	if (pattern === '') { return false }
+	if (!pattern.includes('*') && !pattern.includes('$')) {
+		return path.startsWith(pattern)
+	}
+
+	const anchored = pattern.endsWith('$')
+	const body = anchored ? pattern.slice(0, -1) : pattern
+
+	const expression = body
+		.split('*')
+		.map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+		.join('.*')
+
+	return new RegExp('^' + expression + (anchored ? '$' : '')).test(path)
+}
+
+//
+// Longest matching rule wins; Allow beats Disallow at equal length.
+//
+Robots.isPathAllowed = (rules, path, userAgentToken) => {
+	const group = groupFor(rules, userAgentToken)
+	if (!group) { return true }
+
+	let decision = true
+	let decisionLength = -1
+
+	for (const rule of group.rules) {
+		if (!matchesPattern(rule.path, path)) { continue }
+
+		const length = rule.path.length
+		if (length > decisionLength || (length === decisionLength && rule.allow)) {
+			decision = rule.allow
+			decisionLength = length
+		}
+	}
+
+	return decision
+}
+
+const evict = () => {
+	if (cache.size <= config.robotsMaxOrigins) { return }
+	// Map preserves insertion order, so the oldest key is first.
+	const oldest = cache.keys().next().value
+	cache.delete(oldest)
+}
+
+const getRules = async (origin) => {
+	const cached = cache.get(origin)
+	if (cached && cached.expiresAt > Date.now()) { return cached.rules }
+
+	let rules = ALLOW_ALL
+	let ttl = config.robotsTtlMs
+
+	try {
+		const result = await HttpFetch.fetchText(`${origin}/robots.txt`, {
+			maxBytes: config.maxRobotsBytes,
+		})
+		rules = Robots.parse(result.text)
+	} catch (error) {
+		// No robots.txt, or we could not reach it. Permissive, but cached for a
+		// short while only so a transient outage does not grant a day's pass.
+		rules = ALLOW_ALL
+		ttl = (error && error.kind === 'http' && error.status === 404)
+			? config.robotsNegativeTtlMs
+			: config.robotsErrorTtlMs
+	}
+
+	cache.set(origin, { rules, expiresAt: Date.now() + ttl })
+	evict()
+
+	return rules
+}
+Robots.__getRules = getRules
+
+const originAndPath = (url) => {
+	const parsed = new URL(typeof url === 'string' ? url : url.href)
+	return { origin: parsed.origin, path: parsed.pathname + parsed.search }
+}
+
+//
+// Is this URL allowed by its host's robots.txt?
+//
+Robots.isAllowed = async (url, userAgentToken) => {
+	let parts = null
+	try { parts = originAndPath(url) } catch (error) { return false }
+
+	const rules = await getRules(parts.origin)
+	return Robots.isPathAllowed(rules, parts.path, userAgentToken || config.robotsUaToken)
+}
+
+//
+// Crawl-delay for this host, or null. Capped: a site asking for an hour between
+// requests gets its job failed rather than parked in the queue forever.
+//
+Robots.crawlDelayMs = async (url, userAgentToken) => {
+	let parts = null
+	try { parts = originAndPath(url) } catch (error) { return null }
+
+	const rules = await getRules(parts.origin)
+	const group = groupFor(rules, (userAgentToken || config.robotsUaToken).toLowerCase())
+
+	if (!group || group.crawlDelayMs == null) { return null }
+	return group.crawlDelayMs
+}
+
+Robots.exceedsCrawlDelayCap = (delayMs) => {
+	return delayMs != null && delayMs > config.maxCrawlDelayMs
+}

server/helpers/Schema.js

@@ -0,0 +1,239 @@
+//
+// Verification of the scraper's schema and privileges. Issues NO DDL.
+//
+// This module used to converge the schema at boot with CREATE TABLE / ALTER
+// TABLE. That worked only because dev connects as root (shell.nix exports
+// DB_USER=root); on production, which runs as a least-privilege `app` user, it
+// failed with:
+//
+//   CREATE command denied to user 'app'@'localhost' for table scrape_cache
+//
+// The restricted account is correct - the server ingests untrusted URLs and
+// should not be able to reshape its own tables. So DDL moved out of the
+// application entirely: an admin runs ./migrateDatabase.sh once, and this
+// module only checks that the result is present and usable.
+//
+// Note on why the DDL is REMOVED rather than guarded by an existence check:
+// MariaDB evaluates the CREATE privilege before IF NOT EXISTS, so
+// `CREATE TABLE IF NOT EXISTS` would keep erroring for a DML-only user on every
+// boot even once the table exists.
+//
+
+const db = require('@config/database')
+
+let Schema = module.exports = {}
+
+const REQUIRED_TABLE = 'scrape_cache'
+const REQUIRED_COLUMNS = ['scrape_state', 'scrape_attempts', 'enqueued_at']
+const REQUIRED_INDEX = 'attachment_scrape_state'
+
+//
+// The FULLTEXT index backing Attachment.textSearch's MATCH..AGAINST.
+//
+// Checked by TYPE, not just existence: Prisma's `@@fulltext` is a preview
+// feature and MariaDB is not among its documented databases, so a future
+// migration could plausibly emit a plain BTREE index of the same name. That
+// would make attachment search return nothing at all, with no error anywhere -
+// so it is worth asserting the type explicitly on every boot.
+//
+const REQUIRED_FULLTEXT = { table: 'attachment', index: 'attachment_text_index' }
+
+// The scraper writes and deletes, not just reads. DELETE is genuinely needed:
+// terminal failures drop the shell attachment row (ScrapeQueue), and
+// ScrapeCache.purgeExpired removes expired rows.
+const REQUIRED_PRIVILEGES = ['SELECT', 'INSERT', 'UPDATE', 'DELETE']
+
+// Set by verifyScrapeSchema so the per-request paths can check a flag rather
+// than querying information_schema on every note save.
+let ready = false
+let lastResult = null
+
+Schema.isScrapeSchemaReady = () => ready
+Schema.lastVerification = () => lastResult
+
+// Test seam.
+Schema.__setReady = (value) => { ready = !!value }
+Schema.__reset = () => { ready = false; lastResult = null }
+
+const tableExists = async (table) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.TABLES
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? LIMIT 1`,
+		[table]
+	)
+	return rows.length > 0
+}
+
+const columnExists = async (table, column) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.COLUMNS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`,
+		[table, column]
+	)
+	return rows.length > 0
+}
+
+//
+// Does this index exist AND is it actually FULLTEXT?
+//
+const fulltextIndexExists = async (table, index) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.STATISTICS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?
+			AND INDEX_TYPE = 'FULLTEXT' LIMIT 1`,
+		[table, index]
+	)
+	return rows.length > 0
+}
+
+const indexExists = async (table, index) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.STATISTICS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1`,
+		[table, index]
+	)
+	return rows.length > 0
+}
+
+Schema.__tableExists = tableExists
+Schema.__columnExists = columnExists
+Schema.__indexExists = indexExists
+Schema.__fulltextIndexExists = fulltextIndexExists
+
+//
+// Parse SHOW GRANTS output into the set of privileges that apply to this
+// database. Any user may run SHOW GRANTS FOR CURRENT_USER(), so this needs no
+// privileges of its own.
+//
+// Grant lines look like:
+//   GRANT ALL PRIVILEGES ON *.* TO `root`@`localhost` WITH GRANT OPTION
+//   GRANT SELECT, INSERT, UPDATE, DELETE ON `application`.* TO `app`@`localhost`
+//   GRANT SELECT ON `application`.`attachment` TO `app`@`localhost`
+//   GRANT USAGE ON *.* TO `app`@`localhost`
+//
+const PRIVILEGE_PATTERN = /^GRANT\s+(.+?)\s+ON\s+(\S+)\s+TO\s/i
+
+Schema.parseGrants = (lines, database, tables) => {
+	const granted = new Set()
+
+	// Scope strings that cover our tables. Backticks are optional in output.
+	const strip = (value) => value.replace(/`/g, '')
+	const relevantScopes = new Set([
+		'*.*',
+		`${database}.*`,
+		...tables.map(table => `${database}.${table}`),
+	])
+
+	for (const line of lines) {
+		const match = String(line).match(PRIVILEGE_PATTERN)
+		if (!match) { continue }
+
+		const scope = strip(match[2])
+		if (!relevantScopes.has(scope)) { continue }
+
+		// Drop any column-level qualifiers, e.g. "SELECT (id, url)".
+		const privileges = match[1].replace(/\s*\([^)]*\)/g, '')
+
+		for (const privilege of privileges.split(',')) {
+			const name = privilege.trim().toUpperCase()
+			if (!name || name === 'USAGE') { continue }
+
+			if (name === 'ALL PRIVILEGES' || name === 'ALL') {
+				REQUIRED_PRIVILEGES.forEach(required => granted.add(required))
+				continue
+			}
+
+			granted.add(name)
+		}
+	}
+
+	return granted
+}
+
+const checkPrivileges = async () => {
+	let lines = []
+
+	try {
+		const [rows] = await db.promise().query('SHOW GRANTS FOR CURRENT_USER()')
+		// Each row is an object with a single, server-named column.
+		lines = rows.map(row => Object.values(row)[0])
+	} catch (error) {
+		// If we cannot read our own grants, do not block on a guess - the
+		// object checks above are the real gate.
+		console.log('[schema] could not read grants, skipping privilege check:', error.message)
+		return { missing: [], database: null, currentUser: null }
+	}
+
+	const [dbRows] = await db.promise().query('SELECT DATABASE() AS name, CURRENT_USER() AS who')
+	const database = dbRows[0] && dbRows[0].name
+	const currentUser = dbRows[0] && dbRows[0].who
+
+	const granted = Schema.parseGrants(lines, database, ['attachment', REQUIRED_TABLE])
+
+	return {
+		missing: REQUIRED_PRIVILEGES.filter(privilege => !granted.has(privilege)),
+		database,
+		currentUser,
+	}
+}
+
+// 'app@localhost' -> "'app'@'localhost'", so the remedy is copy-pasteable.
+const quoteUser = (currentUser) => {
+	if (!currentUser) { return '<app user>' }
+
+	const [name, host] = String(currentUser).split('@')
+	return `'${name}'@'${host || 'localhost'}'`
+}
+
+//
+// Check that everything the scraper needs is present. Never mutates anything.
+//
+// Returns { ok, missingObjects, missingPrivileges, remedy }.
+//
+Schema.verifyScrapeSchema = async () => {
+	const missingObjects = []
+
+	if (!await tableExists(REQUIRED_TABLE)) {
+		missingObjects.push(`table ${REQUIRED_TABLE}`)
+	}
+
+	for (const column of REQUIRED_COLUMNS) {
+		if (!await columnExists('attachment', column)) {
+			missingObjects.push(`attachment.${column}`)
+		}
+	}
+
+	if (!await indexExists('attachment', REQUIRED_INDEX)) {
+		missingObjects.push(`attachment index ${REQUIRED_INDEX}`)
+	}
+
+	if (!await fulltextIndexExists(REQUIRED_FULLTEXT.table, REQUIRED_FULLTEXT.index)) {
+		missingObjects.push(`FULLTEXT index ${REQUIRED_FULLTEXT.table}.${REQUIRED_FULLTEXT.index}`)
+	}
+
+	const privileges = await checkPrivileges()
+	const missingPrivileges = privileges.missing || []
+
+	const remedy = []
+	if (missingObjects.length) {
+		remedy.push('run ./migrateDatabase.sh as a database admin')
+	}
+	if (missingPrivileges.length) {
+		// Name the real database and user so this can be pasted straight into
+		// a mysql prompt rather than decoded first.
+		const target = `${privileges.database || '<database>'}.*`
+		remedy.push(
+			`GRANT ${missingPrivileges.join(', ')} ON ${target} TO ${quoteUser(privileges.currentUser)}`
+		)
+	}
+
+	lastResult = {
+		ok: missingObjects.length === 0 && missingPrivileges.length === 0,
+		missingObjects,
+		missingPrivileges,
+		remedy: remedy.join('; '),
+	}
+	ready = lastResult.ok
+
+	return lastResult
+}

server/helpers/SiteScrape.js

@@ -1,174 +1,330 @@
-let SiteScrape = module.exports = {}
+//
-
+// Pure parsing of an already-fetched page. No network I/O, no logging.
 //
 // $ = the cheerio scrape object
 //
+// Everything here is a pure function of ($, url), which is what makes it
+// testable against saved HTML fixtures instead of the live internet.
+//
+
+const config = require('@config/scrape')
+
+let SiteScrape = module.exports = {}
 
 const removeWhitespace = /\s+/g
 
 const commonWords = ['just','start','what','these','how', 'was', 'being','can','way','share','facebook','twitter','reddit','be','have','do','say','get','make','go','know','take','see','come','think','look','want',
-		'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
+	'give','use','find','tell','ask','work','seem','feel','try','leave','call','good','new','first','last','long','great','little','own','other','old',
-		'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
+	'right','big','high','different','small','large','next','early','young','important','few','public','bad','same','able','to','of','in','for','on',
-		'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
+	'with','at','by','from','up','about','into','over','after','the','and','a','that','I','it','not','he','as','you','this','but','his','they','her',
-		'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
+	'she','or','an','will','my','one','all','would','there','their','and','that','but','or','as','if','when','than','because','while','where','after',
-		'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
+	'so','though','since','until','whether','before','although','nor','like','once','unless','now','except','are','also','is','your','its']
-
-SiteScrape.getTitle = ($) => {
-
-	let title = $('title').text().replace(removeWhitespace, " ")
-	return title
 
+const clean = (value, maxLength) => {
+	if (value == null) { return '' }
+	const text = String(value).replace(removeWhitespace, ' ').trim()
+	return maxLength ? text.substring(0, maxLength) : text
 }
 
-//Finds all urls in text, removes duplicates, makes sure they have https://
+// First non-empty wins. Each candidate is a thunk so a missing branch costs
-SiteScrape.getCleanUrls = (textBlock) => {
+// nothing and nothing throws on the way to the answer.
-	//Find all URLs in text
+const firstOf = (candidates) => {
-	//@TODO - Use the process text library for this function
+	for (const candidate of candidates) {
-	const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
+		let value = ''
-	let allUrls = textBlock.match(urlPattern)
+		try { value = clean(typeof candidate === 'function' ? candidate() : candidate) }
-
+		catch (error) { value = '' }
-	if(allUrls == null){
+		if (value) { return value }
-		return []
 	}
+	return ''
+}
 
-	//Every URL needs HTTPS!!!
+const metaContent = ($, selector) => {
-	let foundUrls = []
+	const node = $(selector).first()
-	allUrls.forEach( (item, index) => {
+	return node.length ? node.attr('content') : ''
-		//add protocol if it is missing
+}
-		if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
+
-			allUrls[index] = 'https://'+item
+//
-		}
+// JSON-LD. Malformed JSON-LD is extremely common, so every parse is guarded
-		//convert http to https
+// and a bad block is skipped rather than failing the whole page.
-		if(item.indexOf('http://') >= 0){
+//
-			allUrls[index] = item.replace('http://','https://')
+const LD_TYPES = ['Article', 'NewsArticle', 'BlogPosting', 'WebPage', 'Product', 'VideoObject', 'Recipe']
+
+SiteScrape.getJsonLd = ($) => {
+	const blocks = []
+
+	$('script[type="application/ld+json"]').each((index, element) => {
+		const raw = $(element).contents().text()
+		if (!raw || !raw.trim()) { return }
+
+		let parsed = null
+		try { parsed = JSON.parse(raw) } catch (error) { return }
+
+		// A block may be an object, an array, or wrap its payload in @graph.
+		const flatten = (node) => {
+			if (!node || typeof node !== 'object') { return }
+			if (Array.isArray(node)) { return node.forEach(flatten) }
+			if (Array.isArray(node['@graph'])) { node['@graph'].forEach(flatten) }
+			blocks.push(node)
 		}
+		flatten(parsed)
 	})
 
-	//Remove all duplicates
+	if (!blocks.length) { return null }
-	foundUrls = [...new Set(allUrls)]
 
-	return foundUrls
+	// Prefer a block whose @type looks like page content.
+	const typed = blocks.find(block => {
+		const type = block['@type']
+		const types = Array.isArray(type) ? type : [type]
+		return types.some(entry => LD_TYPES.includes(entry))
+	})
+
+	return typed || blocks[0]
+}
+
+const jsonLdImage = (block) => {
+	if (!block || !block.image) { return '' }
+
+	const image = Array.isArray(block.image) ? block.image[0] : block.image
+	if (!image) { return '' }
+	if (typeof image === 'string') { return image }
+	if (typeof image === 'object') { return image.url || image.contentUrl || '' }
+
+	return ''
+}
+
+//
+// Page title.
+//
+SiteScrape.getTitle = ($) => {
+	const ld = SiteScrape.getJsonLd($)
+
+	return firstOf([
+		() => metaContent($, 'meta[property="og:title"]'),
+		() => metaContent($, 'meta[name="twitter:title"]'),
+		() => ld && (ld.headline || ld.name),
+		() => $('title').first().text(),
+		() => $('h1').first().text(),
+	]).substring(0, 500)
+}
+
+//
+// Page description. The old parser extracted none at all.
+//
+SiteScrape.getDescription = ($) => {
+	const ld = SiteScrape.getJsonLd($)
+
+	return firstOf([
+		() => metaContent($, 'meta[property="og:description"]'),
+		() => metaContent($, 'meta[name="twitter:description"]'),
+		() => metaContent($, 'meta[name="description"]'),
+		() => ld && ld.description,
+	]).substring(0, 1000)
+}
+
+SiteScrape.getSiteName = ($, url) => {
+	return firstOf([
+		() => metaContent($, 'meta[property="og:site_name"]'),
+		() => metaContent($, 'meta[name="application-name"]'),
+		() => {
+			try { return new URL(url).hostname.replace(/^www\./, '') }
+			catch (error) { return '' }
+		},
+	]).substring(0, 255)
+}
+
+//
+// Canonical URL. Used as the base for resolving relative asset paths, which is
+// what the old manual hostname-concatenation got wrong.
+//
+SiteScrape.getCanonicalUrl = ($, base) => {
+	const candidate = firstOf([
+		() => $('link[rel="canonical"]').first().attr('href'),
+		() => metaContent($, 'meta[property="og:url"]'),
+	])
+
+	if (!candidate) { return base || '' }
+
+	try { return new URL(candidate, base || undefined).href }
+	catch (error) { return base || '' }
 }
 
 //Site hostname with https:// eg: https://www.google.com
 SiteScrape.getHostName = (url) => {
+	try {
+		return 'https://' + (new URL(url)).hostname
+	} catch (error) {
+		return ''
+	}
+}
 
-	var hostname = 'https://'+(new URL(url)).hostname;
+//
-	// console.log('hostname', hostname)
+// Image selection.
-	return hostname
+//
+// Extension allowlist plus an allow-if-extensionless rule. The old substring
+// filter required .jpg/.jpeg/.png/.gif anywhere in the string, which silently
+// dropped .webp, .avif, .ico and - more importantly - every extensionless
+// og:image endpoint like /og?title=x, which is what most modern sites serve.
+//
+const IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.avif', '.ico', '.bmp', '.jfif']
+const TRACKING_PIXEL = /(^|[\/_\-.])(1x1|pixel|spacer|blank|transparent|dot)\.(gif|png)/i
+
+const acceptableImagePath = (href) => {
+	let pathname = ''
+	try { pathname = new URL(href).pathname.toLowerCase() }
+	catch (error) { return false }
+
+	const lastSegment = pathname.split('/').pop() || ''
+	const dot = lastSegment.lastIndexOf('.')
+
+	// No extension at all - allow it. An og:image endpoint is the common case.
+	if (dot === -1) { return true }
+
+	const extension = lastSegment.slice(dot)
+
+	if (extension === '.svg') { return !!config.allowSvgThumbnails }
+
+	return IMAGE_EXTENSIONS.includes(extension)
+}
+
+// Resolve against the page's canonical base. Handles protocol-relative
+// (//host/x), root-relative, and path-relative uniformly - no string surgery.
+const resolveImage = (src, base) => {
+	if (!src) { return '' }
+
+	const value = String(src).trim()
+	if (!value || value.startsWith('data:')) { return '' }
+
+	try { return new URL(value, base).href }
+	catch (error) { return '' }
+}
+
+// Largest candidate in a srcset, by width descriptor.
+const largestFromSrcset = (srcset) => {
+	if (!srcset) { return '' }
+
+	let best = ''
+	let bestWidth = -1
+
+	for (const entry of String(srcset).split(',')) {
+		const parts = entry.trim().split(/\s+/)
+		if (!parts[0]) { continue }
+
+		const descriptor = parts[1] || ''
+		const width = descriptor.endsWith('w') ? parseInt(descriptor, 10) : 0
+
+		if (width > bestWidth) {
+			bestWidth = width
+			best = parts[0]
+		}
+	}
+
+	return best
+}
+
+const iconCandidates = ($) => {
+	const icons = []
+
+	// apple-touch-icon is usually the largest and cleanest of the icons.
+	$('link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"]').each((index, element) => {
+		const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
+		icons.push({ href: $(element).attr('href'), weight: 1000 + sizes })
+	})
+
+	$('link[rel="icon"], link[rel="shortcut icon"]').each((index, element) => {
+		const sizes = parseInt($(element).attr('sizes') || '0', 10) || 0
+		icons.push({ href: $(element).attr('href'), weight: sizes })
+	})
+
+	return icons
+		.sort((a, b) => b.weight - a.weight)
+		.map(icon => icon.href)
+}
+
+const inlineImageCandidates = ($) => {
+	const images = []
+
+	$('img').each((index, element) => {
+		const node = $(element)
+
+		// Guarded: the old code read element.attribs.src directly and threw on
+		// an <img> with no attributes.
+		const src = node.attr('src') || node.attr('data-src') || largestFromSrcset(node.attr('srcset'))
+		if (!src) { return }
+
+		const width = parseInt(node.attr('width') || '0', 10) || 0
+		const height = parseInt(node.attr('height') || '0', 10) || 0
+
+		// Tracking pixels and spacers.
+		if ((width > 0 && width <= 1) || (height > 0 && height <= 1)) { return }
+		if (TRACKING_PIXEL.test(src)) { return }
+
+		images.push({ src, area: width * height, order: index })
+	})
+
+	// Biggest first; ties keep document order, so the top of the page wins.
+	return images
+		.sort((a, b) => (b.area - a.area) || (a.order - b.order))
+		.map(image => image.src)
+}
+
+//
+// Ordered candidate list, so "which image" is an explicit priority rather than
+// the old reverse-iterate-and-overwrite loop that only worked by accident.
+//
+SiteScrape.getImageCandidates = ($, url) => {
+	const base = SiteScrape.getCanonicalUrl($, url) || url
+	const ld = SiteScrape.getJsonLd($)
+
+	const raw = [
+		metaContent($, 'meta[property="og:image:secure_url"]'),
+		metaContent($, 'meta[property="og:image:url"]'),
+		metaContent($, 'meta[property="og:image"]'),
+		metaContent($, 'meta[name="twitter:image"]'),
+		metaContent($, 'meta[name="twitter:image:src"]'),
+		jsonLdImage(ld),
+		...inlineImageCandidates($),
+		...iconCandidates($),
+	]
+
+	const seen = new Set()
+	const candidates = []
+
+	for (const entry of raw) {
+		const resolved = resolveImage(entry, base)
+		if (!resolved || seen.has(resolved)) { continue }
+		if (!acceptableImagePath(resolved)) { continue }
+
+		seen.add(resolved)
+		candidates.push(resolved)
+	}
+
+	return candidates
 }
 
 // URL for image that can be downloaded to represent website
 SiteScrape.getDisplayImage = ($, url) => {
-
+	const candidates = SiteScrape.getImageCandidates($, url)
-	const hostname = SiteScrape.getHostName(url)
+	return candidates.length ? candidates[0] : ''
-
-	let metaImg = $('[property="og:image"]')
-	let shortcutIcon = $('[rel="shortcut icon"]')
-	let favicon = $('[rel="icon"]')
-	let randomImg = $('img')
-
-	//Set of images we may want gathered from various places in source
-	let imagesWeWant = []
-	let thumbnail = ''
-
-	//Scrape metadata for page image
-	if(randomImg && randomImg.length > 0){
-
-		let imgSrcs = []
-		for (let i = 0; i < randomImg.length; i++) {
-			imgSrcs.push( randomImg[i].attribs.src )
-		}
-
-		const half = Math.ceil(imgSrcs.length / 2)
-		imagesWeWant = [...imgSrcs.slice(-half), ...imgSrcs.slice(0,half) ]
-
-	}
-	//Grab the shortcut icon
-	if(favicon && favicon[0] && favicon[0].attribs){
-		imagesWeWant.push(favicon[0].attribs.href)
-	}
-	//Grab the shortcut icon
-	if(shortcutIcon && shortcutIcon[0] && shortcutIcon[0].attribs){
-		imagesWeWant.push(shortcutIcon[0].attribs.href)
-	}
-	//Grab the presentation image for the site
-	if(metaImg && metaImg[0] && metaImg[0].attribs){
-		imagesWeWant.unshift(metaImg[0].attribs.content)
-	}
-
-	// console.log(imagesWeWant)
-
-	//Remove everything that isn't an accepted file format
-	for (let i = imagesWeWant.length - 1; i >= 0; i--) {
-
-		let img = String(imagesWeWant[i])
-
-		if(
-			!img.includes('.jpg') && 
-			!img.includes('.jpeg') && 
-			!img.includes('.png') && 
-			!img.includes('.gif')
-		){
-			imagesWeWant.splice(i,1)
-		}
-	}
-
-	//Find if we have absolute thumbnails or not
-	let foundAbsolute = false
-	for (let i = imagesWeWant.length - 1; i >= 0; i--) {
-
-		let img = imagesWeWant[i]
-
-		//Add host name if its not included
-		if(String(img).includes('//') || String(img).includes('http')){
-			foundAbsolute = true
-			break
-		}
-	}
-
-	//Go through all found images. Grab the one closest to the top. Closer is better
-	for (let i = imagesWeWant.length - 1; i >= 0; i--) {
-		
-		let img = imagesWeWant[i]
-
-		if(!String(img).includes('//') && foundAbsolute){
-			continue;
-		}
-
-		//Only add host to images if no absolute images were found
-		if(!String(img).includes('//') ){
-			if(img.indexOf('/') != 0){
-				img = '/' + img
-			}
-			img = hostname + img
-		}
-
-		if(img.indexOf('//') == 0){
-			img = 'https:' + img //Scrape breaks without protocol 
-		}
-			
-		thumbnail = img
-		
-	}
-
-	return thumbnail
 }
 
 // Get all the site text and parse out the words that appear most
 SiteScrape.getKeywords = ($) => {
 
-	let majorContent = ''
+	// Strip chrome and scripts before counting, otherwise navigation labels and
+	// inline JS identifiers dominate the frequency table.
+	const working = $.root().clone()
+	working.find('script, style, noscript, nav, header, footer, aside, form, svg').remove()
 
-	majorContent += $('[class*=content]').text()
+	let source = working.find('article, main, [role=main], [class*=content]').text()
-		.replace(removeWhitespace, " ") //Remove all whitespace
+	if (!source || source.trim().length < 200) {
-		// .replace(/\W\s/g, '') //Remove all non alphanumeric characters
+		source = working.find('body').text() || working.text()
-		.substring(0,6000) //Limit to 6000 characters
+	}
+
+	const majorContent = String(source)
+		.replace(removeWhitespace, ' ')
+		.substring(0, config.maxKeywordSourceChars)
 		.toLowerCase()
-		.replace(/[^A-Za-z0-9- ]/g, '');
+		.replace(/[^A-Za-z0-9- ]/g, '')
-
-
-	console.log(majorContent)
 
 	//Count frequency of each word in scraped text
 	let frequency = {}
@@ -213,7 +369,78 @@ SiteScrape.getKeywords = ($) => {
 	return ''
 }
 
-SiteScrape.getMainText = ($) => {}
+//
+// The plaintext blob stored in attachment.text and indexed by the MySQL
+// FULLTEXT index. Must stay unencrypted for MATCH ... AGAINST to work.
+//
+SiteScrape.buildSearchText = ({ title, description, siteName, keywords }) => {
+	const parts = []
 
+	if (title) { parts.push(title) }
+	if (description) { parts.push(description) }
+	if (siteName) { parts.push(siteName) }
+	if (keywords) { parts.push(keywords) }
 
+	return parts.join('\n').substring(0, config.maxSearchTextBytes)
+}
 
+//
+// One call, everything a link preview needs.
+//
+SiteScrape.parse = ($, url) => {
+	const canonicalUrl = SiteScrape.getCanonicalUrl($, url)
+	const title = SiteScrape.getTitle($)
+	const description = SiteScrape.getDescription($)
+	const siteName = SiteScrape.getSiteName($, url)
+	const keywords = SiteScrape.getKeywords($)
+	const imageCandidates = SiteScrape.getImageCandidates($, url)
+
+	return {
+		title,
+		description,
+		siteName,
+		canonicalUrl,
+		keywords,
+		imageUrl: imageCandidates.length ? imageCandidates[0] : '',
+		imageCandidates,
+		hostname: SiteScrape.getHostName(url),
+		searchText: SiteScrape.buildSearchText({ title, description, siteName, keywords }),
+	}
+}
+
+//
+// Finds all urls in text, removes duplicates, makes sure they have https://
+//
+// DELIBERATELY UNCHANGED. This decides which URLs are detected in note text and
+// what string lands in attachment.url, so altering it would change link
+// behaviour for every existing note - explicitly out of scope for the scraper
+// rewrite. The @TODO below is left in place for the same reason.
+//
+SiteScrape.getCleanUrls = (textBlock) => {
+	//Find all URLs in text
+	//@TODO - Use the process text library for this function
+	const urlPattern = /(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$])/igm
+	let allUrls = textBlock.match(urlPattern)
+
+	if(allUrls == null){
+		return []
+	}
+
+	//Every URL needs HTTPS!!!
+	let foundUrls = []
+	allUrls.forEach( (item, index) => {
+		//add protocol if it is missing
+		if(item.indexOf('https://') == -1 && item.indexOf('http://') == -1){
+			allUrls[index] = 'https://'+item
+		}
+		//convert http to https
+		if(item.indexOf('http://') >= 0){
+			allUrls[index] = item.replace('http://','https://')
+		}
+	})
+
+	//Remove all duplicates
+	foundUrls = [...new Set(allUrls)]
+
+	return foundUrls
+}

server/helpers/UrlSafety.js

@@ -0,0 +1,246 @@
+//
+// URL normalisation and the SSRF gate.
+//
+// Scraped URLs come from untrusted note text and from the /pushmebaby query
+// param, so every URL is validated before a socket is opened - and again after
+// every redirect hop, which is the hop that actually matters (a public URL that
+// 302s to 127.0.0.1 is the classic bypass).
+//
+// Pure except for assertFetchable, which resolves DNS.
+//
+
+const crypto = require('crypto')
+const dns = require('dns').promises
+const net = require('net')
+const config = require('@config/scrape')
+
+let UrlSafety = module.exports = {}
+
+const FETCHABLE_SCHEMES = new Set(['http:', 'https:'])
+
+// Hostname suffixes that never belong to a public site.
+const BLOCKED_SUFFIXES = ['.localhost', '.local', '.internal', '.home.arpa', '.lan']
+
+// Query params that change nothing about the page but would fragment the cache.
+const TRACKING_PARAMS = [
+	/^utm_/i, /^fbclid$/i, /^gclid$/i, /^dclid$/i, /^msclkid$/i,
+	/^mc_(e|c)id$/i, /^igshid$/i, /^ref_src$/i, /^_hs(enc|mi)$/i, /^vero_/i,
+]
+
+class SafetyError extends Error {
+	constructor(reason, message) {
+		super(message || reason)
+		this.name = 'SafetyError'
+		this.reason = reason
+	}
+}
+UrlSafety.SafetyError = SafetyError
+
+//
+// Parse to a URL object, or null. Does NOT mutate anything that would change
+// which resource is addressed - callers may store the result.
+//
+UrlSafety.parse = (raw, base) => {
+	if (!raw || typeof raw !== 'string') { return null }
+
+	let url = null
+	try {
+		url = new URL(raw.trim(), base || undefined)
+	} catch (error) {
+		return null
+	}
+
+	if (!FETCHABLE_SCHEMES.has(url.protocol)) { return null }
+	if (url.href.length > config.maxUrlLength) { return null }
+
+	return url
+}
+
+//
+// Canonical form used ONLY for cache keys. Never store this - it deliberately
+// discards tracking params and case, which would change what the user wrote.
+//
+UrlSafety.normalizeForKey = (raw) => {
+	const url = UrlSafety.parse(raw)
+	if (!url) { return null }
+
+	url.hash = ''
+	url.username = ''
+	url.password = ''
+
+	// Trailing dot on the host is legal DNS but splits the cache key.
+	if (url.hostname.endsWith('.')) {
+		url.hostname = url.hostname.slice(0, -1)
+	}
+	url.hostname = url.hostname.toLowerCase()
+
+	// new URL already strips the default port, but be explicit.
+	if ((url.protocol === 'http:' && url.port === '80') ||
+		(url.protocol === 'https:' && url.port === '443')) {
+		url.port = ''
+	}
+
+	for (const key of [...url.searchParams.keys()]) {
+		if (TRACKING_PARAMS.some(pattern => pattern.test(key))) {
+			url.searchParams.delete(key)
+		}
+	}
+	url.searchParams.sort()
+
+	if (url.pathname === '') { url.pathname = '/' }
+
+	return url
+}
+
+// sha256 of the normalised href. This is scrape_cache.url_hash - char(64),
+// because a UNIQUE index on varchar(1000) utf8mb4 would be 4000 bytes and
+// blow InnoDB's 3072-byte key limit.
+UrlSafety.urlKey = (raw) => {
+	const url = UrlSafety.normalizeForKey(raw)
+	if (!url) { return null }
+	return crypto.createHash('sha256').update(url.href).digest('hex')
+}
+
+//
+// True when two URLs address the same resource modulo scheme, "www." and a
+// trailing slash.
+//
+// Load-bearing: scanTextForWebsites diffs the URLs currently in a note against
+// the URLs already stored as attachments. Raw string equality would treat
+// http://foo.com and https://foo.com as different resources and delete-then-
+// rescrape the row on every save.
+//
+UrlSafety.sameResource = (a, b) => {
+	const left = UrlSafety.normalizeForKey(a)
+	const right = UrlSafety.normalizeForKey(b)
+	if (!left || !right) { return false }
+
+	const host = (url) => url.hostname.replace(/^www\./, '')
+	const path = (url) => url.pathname.replace(/\/+$/, '')
+
+	return host(left) === host(right) &&
+		path(left) === path(right) &&
+		left.search === right.search
+}
+
+UrlSafety.registrableHost = (hostname) => {
+	return String(hostname || '').toLowerCase().replace(/^www\./, '').replace(/\.$/, '')
+}
+
+//
+// Private / non-routable address detection.
+//
+// Covers the ranges that matter for SSRF against this app: loopback, RFC1918,
+// link-local (including the 169.254.169.254 cloud metadata endpoint), CGNAT,
+// and the IPv6 equivalents.
+//
+UrlSafety.isPrivateIp = (address) => {
+	if (!address) { return true }
+
+	const version = net.isIP(address)
+	if (version === 0) { return true } // not an IP - caller resolves first
+
+	if (version === 4) { return isPrivateIpv4(address) }
+	return isPrivateIpv6(address)
+}
+
+const isPrivateIpv4 = (address) => {
+	const parts = address.split('.').map(Number)
+	if (parts.length !== 4 || parts.some(n => !Number.isInteger(n) || n < 0 || n > 255)) {
+		return true
+	}
+	const [a, b] = parts
+
+	if (a === 0) { return true }                            // 0.0.0.0/8 "this host"
+	if (a === 10) { return true }                           // RFC1918
+	if (a === 127) { return true }                          // loopback
+	if (a === 169 && b === 254) { return true }             // link-local + metadata
+	if (a === 172 && b >= 16 && b <= 31) { return true }    // RFC1918
+	if (a === 192 && b === 168) { return true }             // RFC1918
+	if (a === 192 && b === 0) { return true }               // 192.0.0/24 + 192.0.2/24
+	if (a === 198 && (b === 18 || b === 19)) { return true } // benchmarking
+	if (a === 100 && b >= 64 && b <= 127) { return true }   // CGNAT
+	if (a === 198 && b === 51) { return true }              // TEST-NET-2
+	if (a === 203 && b === 0) { return true }               // TEST-NET-3
+	if (a >= 224) { return true }                           // multicast + reserved
+
+	return false
+}
+
+const isPrivateIpv6 = (address) => {
+	const lower = address.toLowerCase().replace(/^\[|\]$/g, '')
+
+	// Unwrap v4-mapped (::ffff:127.0.0.1) and NAT64 (64:ff9b::/96) before
+	// judging, otherwise ::ffff:127.0.0.1 reads as "some v6 address".
+	const mapped = lower.match(/^(?:::ffff:|64:ff9b::)(\d+\.\d+\.\d+\.\d+)$/)
+	if (mapped) { return isPrivateIpv4(mapped[1]) }
+
+	if (lower === '::' || lower === '::1') { return true }   // unspecified, loopback
+	if (/^f[cd]/.test(lower)) { return true }               // fc00::/7 unique-local
+	if (/^fe[89ab]/.test(lower)) { return true }            // fe80::/10 link-local
+	if (/^ff/.test(lower)) { return true }                  // multicast
+
+	return false
+}
+
+//
+// The gate. Throws SafetyError; resolves to the vetted addresses on success.
+//
+// Known limitation (documented, accepted): this resolves DNS and then hands the
+// hostname to fetch, which resolves again. A hostile 0-TTL record can answer
+// differently the second time. Closing that properly needs a custom undici
+// Agent with connect.lookup - deliberately deferred, see the plan's Phase 5.
+// Residual risk is blind SSRF only: no credentials are sent and no response
+// body is returned to whoever planted the URL.
+//
+UrlSafety.assertFetchable = async (raw) => {
+	const url = raw instanceof URL ? raw : UrlSafety.parse(raw)
+
+	if (!url) {
+		throw new SafetyError('bad_url', `Not a fetchable URL: ${raw}`)
+	}
+	if (!FETCHABLE_SCHEMES.has(url.protocol)) {
+		throw new SafetyError('bad_scheme', `Refusing scheme ${url.protocol}`)
+	}
+	if (url.username || url.password) {
+		throw new SafetyError('credentials_in_url', 'Refusing URL with embedded credentials')
+	}
+
+	const hostname = url.hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase()
+
+	if (!hostname) {
+		throw new SafetyError('bad_url', 'URL has no host')
+	}
+	if (hostname === 'localhost' || BLOCKED_SUFFIXES.some(suffix => hostname.endsWith(suffix))) {
+		throw new SafetyError('blocked_host', `Refusing internal host ${hostname}`)
+	}
+
+	// Literal IP in the URL - no DNS needed.
+	if (net.isIP(hostname)) {
+		if (UrlSafety.isPrivateIp(hostname)) {
+			throw new SafetyError('private_ip', `Refusing non-routable address ${hostname}`)
+		}
+		return [hostname]
+	}
+
+	let resolved = []
+	try {
+		resolved = await dns.lookup(hostname, { all: true, verbatim: true })
+	} catch (error) {
+		throw new SafetyError('dns_failed', `Could not resolve ${hostname}: ${error.code || error.message}`)
+	}
+
+	if (!resolved.length) {
+		throw new SafetyError('dns_failed', `No addresses for ${hostname}`)
+	}
+
+	// Every answer must be public. One private address is enough to refuse -
+	// otherwise a round-robin record gets us some of the time.
+	for (const entry of resolved) {
+		if (UrlSafety.isPrivateIp(entry.address)) {
+			throw new SafetyError('private_ip', `${hostname} resolves to non-routable ${entry.address}`)
+		}
+	}
+
+	return resolved.map(entry => entry.address)
+}

server/index.js

@@ -8,11 +8,17 @@ const ports = {
 }
 
 //Allow user of @ in in require calls. Config in package.json
-require('module-alias/register')
+// Point module-alias at the repo root explicitly. The bare /register entry
+// locates package.json by walking up from its own directory, which lands
+// inside .pnpm/ under pnpm's symlinked store and fails to find it.
+require('module-alias')(require('path').join(__dirname, '..'))
 
 //Auth helper, used for decoding users web token
 let Auth = require('@helpers/Auth')
 
+//Note model, used to authorise socket room joins
+let Note = require('@models/Note')
+
 //Helmet adds additional security to express server
 const helmet = require('helmet')
 
@@ -48,6 +54,14 @@ global.SocketIo = io
 
 let noteDiffs = {}
 
+//
+// Room names are namespaced. Previously note rooms were keyed on the bare
+// note_raw_text.id and user rooms on the bare user.id, sharing one flat
+// keyspace -- so user 7 and note_raw_text 7 were the same room.
+//
+const userRoom = userId => `user:${userId}`
+const noteRoom = rawTextId => `note:${rawTextId}`
+
 io.on('connection', function(socket){
 
 	//When a user connects, add them to their own room
@@ -56,11 +70,11 @@ io.on('connection', function(socket){
 	socket.on('user_connect', token => {
 		Auth.decodeToken(token)
 		.then(userData => {
-			socket.join(userData.userId)
+			socket.join(userRoom(userData.userId))
 
 			//Track active logged in user accounts
-			const usersInRoom = io.sockets.adapter.rooms[userData.userId]
+			const usersInRoom = io.sockets.adapter.rooms[userRoom(userData.userId)]
-			io.to(userData.userId).emit('update_active_user_count', usersInRoom.length)
+			io.to(userRoom(userData.userId)).emit('update_active_user_count', usersInRoom.length)
 
 		}).catch(error => {
 			//Don't add user to room if they are not logged in
@@ -71,11 +85,11 @@ io.on('connection', function(socket){
 	socket.on('get_active_user_count', token => {
 		Auth.decodeToken(token)
 		.then(userData => {
-			socket.join(userData.userId)
+			socket.join(userRoom(userData.userId))
 
 			//Track active logged in user accounts
-			const usersInRoom = io.sockets.adapter.rooms[userData.userId]
+			const usersInRoom = io.sockets.adapter.rooms[userRoom(userData.userId)]
-			io.to(userData.userId).emit('update_active_user_count', usersInRoom.length)
+			io.to(userRoom(userData.userId)).emit('update_active_user_count', usersInRoom.length)
 
 		}).catch(error => {
 			// console.log(error)
@@ -104,67 +118,106 @@ io.on('connection', function(socket){
 		})
 	})
 
-	socket.on('join_room', rawTextId => {
+	//
-		// Join user to rawtextid room when they enter
+	// The client asks for a NOTE, not a room. The server authorises the request
-		socket.join(rawTextId)
+	// against the session and derives the room itself, so a client can no longer
+	// name an arbitrary room and be handed its buffered diffs.
+	//
+	socket.on('join_room', ({ noteId, token } = {}) => {
 
-		//If there are past diffs for this note, send them to the user
+		Auth.decodeToken(token)
-		if(noteDiffs[rawTextId] != undefined){
+		.then(userData => {
 
-			//Sort all note diffs by when they were created.
+			return Note.getRawTextIdForUser(noteId, userData.userId)
-			noteDiffs[rawTextId].sort((a,b) => { return a.time - b.time })
+			.then(rawTextId => {
 
-			//Emit all sorted diffs to user
+				// No claim on this note -> no room, no diffs
-			socket.emit('past_diffs', noteDiffs[rawTextId])
+				if(rawTextId == null){ return }
-		}
 
-		const usersInRoom = io.sockets.adapter.rooms[rawTextId]
+				const room = noteRoom(rawTextId)
-		if(usersInRoom){
-			//Update users in room count
-			io.to(rawTextId).emit('update_user_count', usersInRoom.length)
-		}
-	})
 
-	socket.on('leave_room', roomId => {
+				// Remember what this socket is allowed into, so leave_room and
-		socket.leave(roomId)
+				// note_diff never have to trust a client-supplied room again
-		// console.log('User Left room')
+				socket.authorizedRooms = socket.authorizedRooms || {}
+				socket.authorizedRooms[noteId] = { room, rawTextId }
 
-		const usersInRoom = io.sockets.adapter.rooms[roomId]
+				socket.join(room)
-		if(usersInRoom){
-			// console.log('Users in room', usersInRoom.length)
-			io.to(roomId).emit('update_user_count', usersInRoom.length)
-		}
 
-	})
+				//If there are past diffs for this note, send them to the user
+				if(noteDiffs[rawTextId] != undefined){
 
-	socket.on('note_diff', data => {
+					//Sort all note diffs by when they were created.
+					noteDiffs[rawTextId].sort((a,b) => { return a.time - b.time })
 
-		//Log each diff for note
+					//Emit all sorted diffs to user
-		const noteId = data.id
+					socket.emit('past_diffs', noteDiffs[rawTextId])
-		delete data.id
+				}
-		if(noteDiffs[noteId] == undefined){ noteDiffs[noteId] = [] }
-		data.time = +new Date
 
-		noteDiffs[noteId].push(data)
+				const usersInRoom = io.sockets.adapter.rooms[room]
-
+				if(usersInRoom){
-		// Go over each user in this note-room
+					//Update users in room count
-		io.in(noteId).clients((error, clients) => {
+					io.to(room).emit('update_user_count', usersInRoom.length)
-			if (error) throw error;
-
-			//Go through each client in note-room and send them the diff
-			clients.forEach(socketId => {
-				// only send off diff if user
-				if(socketId != socket.id){
-					io.to(socketId).emit('incoming_diff', data)
 				}
 			})
-
+		})
-		});
+		.catch(() => {
-
+			//Not logged in, or no access. Join nothing.
+		})
 	})
 
-	socket.on('truncate_diffs_at_save', checkpoint => {
+	socket.on('leave_room', ({ noteId } = {}) => {
 
+		// Only rooms this socket was actually authorised into
+		const entry = socket.authorizedRooms && socket.authorizedRooms[noteId]
+		if(!entry){ return }
+
+		socket.leave(entry.room)
+		delete socket.authorizedRooms[noteId]
+
+		const usersInRoom = io.sockets.adapter.rooms[entry.room]
+		if(usersInRoom){
+			io.to(entry.room).emit('update_user_count', usersInRoom.length)
+		}
+	})
+
+	socket.on('note_diff', (data = {}) => {
+
+		// Room comes from what this socket was authorised into, never from the
+		// payload
+		const entry = socket.authorizedRooms && socket.authorizedRooms[data.noteId]
+		if(!entry){ return }
+
+		const room = entry.room
+		const rawTextId = entry.rawTextId
+
+		//Log each diff for note
+		if(noteDiffs[rawTextId] == undefined){ noteDiffs[rawTextId] = [] }
+
+		// Carry the raw-text id through. This used to `delete data.id`, stripping
+		// the only identifier before storing and broadcasting -- so a client with
+		// several notes open on one socket could not tell which note a patch
+		// belonged to, and applied every one to whichever editor it found first.
+		const payload = Object.assign({}, data, { rawTextId, time: +new Date })
+		delete payload.noteId
+
+		noteDiffs[rawTextId].push(payload)
+
+		// Everyone in the note room except the sender
+		socket.to(room).emit('incoming_diff', payload)
+	})
+
+	socket.on('truncate_diffs_at_save', (checkpoint = {}) => {
+
+		const entry = socket.authorizedRooms && socket.authorizedRooms[checkpoint.noteId]
+		if(!entry){ return }
+		checkpoint.rawTextId = entry.rawTextId
+
+		// NOTE: the comparison below is `diffSet[i].hash == checkpoint`, comparing
+		// a hash string against the whole checkpoint object, so it never matches
+		// and sliceTo stays 0 -- meaning the entire diff set is discarded at every
+		// save. That happens to be the fail-safe outcome (the saved text becomes
+		// the new baseline), and the correct slice direction for a real fix is
+		// ambiguous, so it is deliberately left alone rather than guessed at.
 		let diffSet = noteDiffs[checkpoint.rawTextId]
 		if(diffSet && diffSet.length > 0){
 
@@ -292,10 +345,72 @@ app.use('/api/quick-note', quickNote)
 var metricTracking = require('@routes/metrictrackingController')
 app.use('/api/metric-tracking', metricTracking)
 
-//Output running status
+//
-app.listen(ports.express, () => { 
+// Scraper startup.
-	console.log(`Express: Listening on port ${ports.express}!`)
+//
-})
+// The application deliberately holds no DDL privileges, so it does not create
+// its own schema - verifyScrapeSchema() only checks that an admin has already
+// run ./migrateDatabase.sh and that the required DML grants are present.
+//
+// When anything is missing the scraper stays off rather than failing an INSERT
+// on every note save. The listener binds either way: the API must not depend on
+// link previews being available.
+//
+const Schema = require('@helpers/Schema')
+const ScrapeQueue = require('@workers/ScrapeQueue')
+
+Schema.verifyScrapeSchema()
+	.then(result => {
+		if (!result.ok) {
+			const missing = [
+				...result.missingObjects,
+				...result.missingPrivileges.map(privilege => `privilege ${privilege}`),
+			].join(', ')
+
+			console.log(`Scraper: DISABLED - missing ${missing}. To fix: ${result.remedy}`)
+			return null
+		}
+
+		// Say so positively. Previously a healthy scraper logged nothing, so
+		// "started" and "silently disabled" looked identical in pm2 logs --
+		// which is how a dead scraper went unnoticed on production.
+		console.log('Scraper: ready')
+
+		ScrapeQueue.start()
+
+		// The queue lives in memory, so a pm2 reload loses whatever was in it.
+		// Every enqueued URL already has an attachment row, which makes the
+		// work re-derivable without any extra table.
+		return ScrapeQueue.recoverPending()
+	})
+	.then(recovery => {
+		if (recovery && recovery.requeued) {
+			console.log(`Scraper: re-queued ${recovery.requeued} pending URL(s), retired ${recovery.retired}`)
+		}
+	})
+	.catch(error => {
+		// A scraper that cannot start must not stop the API from serving.
+		console.log('Scraper: startup check failed, link previews disabled this boot:', error.message)
+	})
+	.finally(() => {
+		//Output running status
+		app.listen(ports.express, () => {
+			console.log(`Express: Listening on port ${ports.express}!`)
+		})
+	})
+
+//
+// pm2 reload sends SIGINT. Stop accepting new scrape work and let in-flight
+// requests abort via their own AbortControllers; rows left mid-fetch are
+// recovered on the next boot.
+//
+const shutdown = (signal) => {
+	console.log(`Received ${signal}, stopping scrape queue`)
+	try { ScrapeQueue.stop() } catch (error) { /* already down */ }
+	process.exit(0)
+}
+process.on('SIGINT', () => shutdown('SIGINT'))
+process.on('SIGTERM', () => shutdown('SIGTERM'))
 
 //
 //Error handlers

server/models/Attachment.js

@@ -6,15 +6,30 @@ const cs = require('@helpers/CryptoString')
 let Attachment = module.exports = {}
 
 const cheerio = require('cheerio')
-const rp = require('request-promise')
-const request = require('request')
 const fs = require('fs')
+const path = require('path')
+const crypto = require('crypto')
 
 const gm = require('gm')
 
 const tesseract = require("node-tesseract-ocr")
+
+// Left as-is deliberately: the upload / OCR / thumbnail code below resolves
+// against it, and it is correct under pm2 (cwd = server/). Not touching it
+// keeps the upload path byte-identical.
 const filePath = '../staticFiles/'
 
+// Absolute equivalent, used by the scrape path only. filePath is CWD-relative,
+// so under Jest (cwd = repo root) scrape downloads landed in a sibling of the
+// repo rather than in staticFiles/. This resolves to the same directory as
+// filePath under pm2.
+const scrapeFilePath = path.join(__dirname, '..', '..', 'staticFiles') + path.sep
+
+const config = require('@config/scrape')
+const UrlSafety = require('@helpers/UrlSafety')
+const HttpFetch = require('@helpers/HttpFetch')
+const Realtime = require('@helpers/Realtime')
+
 // Attachment.migrateOld
 
 Attachment.textSearch = (userId, searchTerm) => {
@@ -263,264 +278,235 @@ Attachment.generateThumbnail = (fileName) => {
 	})
 }
 
-//Scans text for websites, returns all attachments
+//
-Attachment.scanTextForWebsites = (io, userId, noteId, noteText) => {
+// Diff the URLs currently in a note against the attachments already stored for
-	return new Promise((resolve, reject) => {
+// it, soft-delete the ones that vanished, and enqueue the new ones.
+//
+// Returns immediately with the text already known from cache; freshly scraped
+// text arrives asynchronously via the worker. Nothing here waits on the network.
+//
+Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
 
-		let solrAttachmentText = '' //Final searchable scrape text for note
+	const ScrapeQueue = require('@workers/ScrapeQueue')
+	const Schema = require('@helpers/Schema')
 
-		if(noteText.length == 0){ resolve(solrAttachmentText) }
+	// Fixed: the old version called resolve() without returning, so an empty
+	// note fell through and ran the DB lookup anyway.
+	if (!noteText || noteText.length == 0) { return Promise.resolve('') }
 
-		Attachment.urlForNote(userId, noteId).then(attachments => {
+	// When the scrape schema is absent the new columns do not exist, so
+	// creating a pending row would fail on every save. The diff and soft-delete
+	// below still run - they touch no new columns - so removing a link from a
+	// note behaves exactly as before.
+	const scrapeAvailable = Schema.isScrapeSchemaReady()
 
-			//Pull all the URLs out of the text
+	let knownText = ''
-			let foundUrls = SiteScrape.getCleanUrls(noteText)
+
+	return Attachment.urlForNote(userId, noteId)
+		.then(attachments => {
+
+			const foundUrls = SiteScrape.getCleanUrls(noteText)
+			const remaining = foundUrls.slice()
 
-			//Go through each saved URL, remove new URLs from saved URLs
-			//If a URL is not found, delete it
 			attachments.forEach(attachment => {
-				//URL already scraped, push text and continue
-				let urlIndex = foundUrls.indexOf( attachment.url )
 
-				if(urlIndex != -1){
+				// Compared on normalised keys rather than raw strings, so an
-					solrAttachmentText += attachment.text
+				// http/https or www. variant is not treated as a different
-					foundUrls.splice(urlIndex, 1) //Remove existing from set of found
+				// resource and needlessly deleted and re-scraped.
+				const index = remaining.findIndex(url =>
+					url === attachment.url || UrlSafety.sameResource(url, attachment.url))
+
+				if (index !== -1) {
+					knownText += attachment.text || ''
+					remaining.splice(index, 1)
 				} else {
-					//If existing attachment is not found in note, remove it
+					// Unchanged behaviour: link attachments are hidden, not deleted.
 					Attachment.delete(userId, attachment.id, true)
 				}
 			})
 
-			//No newly scraped URLs, resolve with looked up attachment text
+			if (!remaining.length) { return knownText }
-			if(foundUrls == null || foundUrls.length == 0){
-				return resolve(solrAttachmentText)
-			}
 
-			//Process the remaining URLs into attachments
+			// No schema, no previews. Return what is already cached rather than
-			Attachment.scrapeUrlsCreateAttachments(userId, noteId, foundUrls).then( freshlyScrapedText => {
+			// attempting inserts that cannot succeed.
+			if (!scrapeAvailable) { return knownText }
 
-				//Once everything is done being scraped, emit new attachment events
+			// Create a shell row per new URL, then hand off to the worker.
-				SocketIo.to(userId).emit('update_counts')
+			return Promise.all(remaining.map(url =>
-
+				Attachment.createPendingUrlAttachment(userId, noteId, url)
-				// Tell user to update attachments with scraped text
+					.then(attachmentId => ({ userId, noteId, attachmentId, url }))
-				SocketIo.to(userId).emit('update_note_attachments')
+					.catch(error => {
-
+						console.log('[scrape] could not create attachment row', url, error.message)
-				solrAttachmentText += freshlyScrapedText
+						return null
-				resolve(solrAttachmentText)
-			})
-			.catch(console.log)
-		})
-	})
-}
-
-//Return scraped text from each URL
-Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
-	return new Promise((resolve, reject) => {
-
-		if(foundUrls == null || foundUrls.length == 0){ 
-			return resolve('')
-		}
-
-		console.log('About to scrape')
-		console.log(foundUrls)
-
-		let processedCount = 0
-		let scrapedText = ''
-
-		//Process each URL passd to function, a DB entry will be created for each scrape
-		foundUrls.forEach(url => {
-			Attachment.processUrl(userId, noteId, url).then( freshlyScrapedText => {
-				
-				scrapedText += freshlyScrapedText
-				processedCount ++
-
-				//All URLs have been scraped, return data
-				if(processedCount == foundUrls.length){
-					console.log('All urls scraped')
-					return resolve(scrapedText)
-				}
-			})
-			.catch(error => {
-				console.log('Site Scrape error', error)
-			})
-		})
-	})
-}
-
-Attachment.downloadFileFromUrl = (url) => {
-	
-
-	return new Promise((resolve, reject) => {
-
-			if(!url){
-				return resolve(null)
-			}
-
-			const random = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
-			let extension = ''
-			let fileName = random+'_scrape'
-			let thumbPath = 'thumb_'+fileName
-
-			console.log('Scraping image url', url)
-
-			console.log('Getting ready to scrape ', url)
-
-			request(url)
-				.on('error', error => {
-					console.log(error)
-					resolve(null)
-				})
-				.on('response', res => {
-					console.log(res.statusCode)
-					console.log(res.headers['content-type'])
-					//Get mime type from header content type
-					// extension = '.'+String(res.headers['content-type']).split('/').pop()
-				})
-				.pipe(fs.createWriteStream(filePath+thumbPath))
-				.on('close', () => {
-
-					//resize image if its real big
-					gm(filePath+thumbPath)
-					.resize(550) //Resize to width of 550 px 
-					.quality(85) //compression level 0 - 100 (best)
-					.write(filePath+thumbPath, function (err) {
-						if(err){ 
-							console.log(err) 
-							return resolve(null)
-						}
-
-						console.log('Saved Image')
-						return resolve(fileName)
 					})
+			))
+			.then(jobs => {
+				const valid = jobs.filter(Boolean)
 
-				})
+				// Fire and forget - the note save must not wait on this.
+				ScrapeQueue.enqueueUrls(valid)
+					.catch(error => console.log('[scrape] enqueue failed', error.message))
+
+				return knownText
+			})
+		})
+		// The old version had no catch here at all.
+		.catch(error => {
+			console.log('[scrape] scanTextForWebsites failed', error.message)
+			return knownText
+		})
+}
+
+//
+// Create the shell attachment row for a URL and return its id.
+//
+// Returns the id BEFORE any network work begins. The old code armed a 5s
+// setTimeout calling request.cancel() synchronously, before this INSERT had
+// resolved - so a slow INSERT meant the timer fired against a null reference
+// and threw inside a bare timer callback, taking the process down.
+//
+Attachment.createPendingUrlAttachment = (userId, noteId, url) => {
+	const created = Math.round((+new Date) / 1000)
+
+	return db.promise()
+		.query(
+			`INSERT INTO attachment
+				(note_id, user_id, attachment_type, text, url, last_indexed, file_location, scrape_state, enqueued_at)
+			 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			[noteId, userId, 1, url, url, created, null, 0, created]
+		)
+		.then(([result]) => result.insertId)
+}
+
+//
+// Scrape a set of URLs for a note and resolve with their combined text.
+//
+// Kept for the bookmarklet path. The old implementation resolved only when a
+// completion counter reached foundUrls.length, so a single rejection meant the
+// promise never settled and the caller hung forever. allSettled inside the
+// worker removes the counter entirely.
+//
+Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
+
+	const ScrapeQueue = require('@workers/ScrapeQueue')
+	const Schema = require('@helpers/Schema')
+
+	if (!foundUrls || !foundUrls.length) { return Promise.resolve('') }
+	if (!Schema.isScrapeSchemaReady()) { return Promise.resolve('') }
+
+	return Promise.all(foundUrls.map(url =>
+		Attachment.createPendingUrlAttachment(userId, noteId, url)
+			.then(attachmentId => ({ userId, noteId, attachmentId, url }))
+			.catch(error => {
+				console.log('[scrape] could not create attachment row', url, error.message)
+				return null
+			})
+	))
+	.then(jobs => ScrapeQueue.enqueueUrls(jobs.filter(Boolean)))
+	.then(texts => texts.join(''))
+}
+
+//
+// Download and store a preview image.
+//
+// The old downloadFileFromUrl had no timeout, no size limit, no User-Agent and
+// no status check: it piped whatever came back straight into a write stream, so
+// a 404 HTML error page was written to disk and handed to gm. It also wrote the
+// file before validating anything, which is why staticFiles/ accumulated
+// zero-byte orphans.
+//
+Attachment.downloadThumbnail = (imageUrl, pageUrl) => {
+
+	if (!imageUrl) { return Promise.resolve(null) }
+
+	// crypto.randomBytes rather than two concatenated Math.random() calls.
+	const fileName = crypto.randomBytes(16).toString('hex') + '_scrape'
+	const fullPath = scrapeFilePath + fileName
+	const thumbPath = scrapeFilePath + 'thumb_' + fileName
+	const tempPath = fullPath + '.tmp'
+
+	return HttpFetch.fetchBinary(imageUrl, {
+		pageUrl,
+		timeoutMs: config.imageTimeoutMs,
+		maxBytes: config.maxImageBytes,
+	})
+	.then(result => {
+
+		// Status and content-type were both gated inside fetchBinary, so by
+		// here we are holding bytes that really are an image.
+		if (!result.buffer || !result.buffer.length) { return null }
+		if (!magicBytesMatch(result.buffer, result.mime)) {
+			console.log('[scrape] image bytes do not match declared type', result.mime, imageUrl)
+			return null
+		}
+
+		// Write to a temp path, then rename, so a failure cannot leave a
+		// half-written file behind for the client to serve.
+		return fs.promises.writeFile(tempPath, result.buffer)
+			.then(() => fs.promises.rename(tempPath, fullPath))
+			.then(() => resizeToThumbnail(fullPath, thumbPath))
+			.then(() => fileName)
+			.catch(error => {
+				console.log('[scrape] could not store image', imageUrl, error.message)
+				return cleanupFiles([tempPath, fullPath, thumbPath]).then(() => null)
+			})
+	})
+	.catch(error => {
+		console.log('[scrape] image fetch failed', imageUrl, error.kind || error.message)
+		return null
 	})
 }
 
-Attachment.processUrl = (userId, noteId, url) => {
+//
+// Resize via gm. Degrades rather than failing: gm (and tesseract) are not in
+// shell.nix buildInputs, so the binary is absent in dev - the old code returned
+// null here and left the downloaded file orphaned on disk. Copying the
+// full-size image to the thumbnail path keeps the preview working.
+//
+const resizeToThumbnail = (fullPath, thumbPath) => {
+	return new Promise(resolve => {
+		gm(fullPath)
+			.resize(550)   //Resize to width of 550 px
+			.quality(85)   //compression level 0 - 100 (best)
+			.write(thumbPath, error => {
+				if (!error) { return resolve(true) }
 
-	const scrapeTime = 5*1000; 
+				console.log('[scrape] gm unavailable or failed, using full size as thumbnail')
-
+				fs.promises.copyFile(fullPath, thumbPath)
-	return new Promise((resolve, reject) => {
+					.then(() => resolve(true))
-
+					.catch(() => resolve(false))
-
-		const options = {
-			uri: url,
-			simple: true,
-			timeout: scrapeTime,
-			headers: {
-				'User-Agent':'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' //Simulate google headers
-			},
-			transform: function (body) {
-				return cheerio.load(body);
-			}
-		}
-
-		let requestTimeout = null
-		let thumbnail = null
-		let request = null
-		let created = Math.round((+new Date)/1000)
-		let insertedId = null
-
-		//Create a shell attachment for each URL, put in processing state
-		db.promise()
-		.query(`INSERT INTO attachment 
-			(note_id, user_id, attachment_type, text, url, last_indexed, file_location) 
-			VALUES (?, ?, ?, ?, ?, ?, ?)`, 
-			[noteId, userId, 1, url, url, created, null])
-		.then((rows, fields) => {
-			//Set two bigger variables then return request for processing
-			request = rp(options)
-			insertedId = rows[0].insertId
-
-			return request
-		})
-		.then($ => {
-
-			//Clear timeout that would end this function
-			clearTimeout(requestTimeout)
-
-			// let header = $('h1').text().replace(removeWhitespace, " ")
-			// desiredSearchText += header + "\n"
-
-			const pageTitle = SiteScrape.getTitle($)
-
-			const hostname = SiteScrape.getHostName(url)
-
-			const thumbnail = SiteScrape.getDisplayImage($, url)
-
-			const keywords = SiteScrape.getKeywords($)
-
-			var desiredSearchText = ''
-			desiredSearchText += pageTitle
-			if(keywords){
-				desiredSearchText += "\n " + keywords
-			}
-
-			console.log('Results from site scrape-------------')
-			console.log({
-				pageTitle,
-				hostname,
-				thumbnail,
-				keywords
 			})
-			
-
-			// throw new Error('Ending this function early.')
-
-			
-			// console.log('TexT Scraped')
-			// console.log(desiredSearchText)
-
-			created = Math.round((+new Date)/1000)
-
-			//Scrape URL for thumbnail - take filename and save in attachment
-			Attachment.downloadFileFromUrl(thumbnail)
-			.then(thumbnailFilename => {
-
-				//Update text and thumbnail filename
-				created = Math.round((+new Date)/1000)
-				db.promise()
-				.query(`UPDATE attachment SET 
-					text = ?,
-					last_indexed = ?,
-					file_location = ?
-					WHERE id = ?
-				`, [desiredSearchText, created, thumbnailFilename, insertedId])
-				.then((rows, fields) => {
-					resolve(desiredSearchText) //Return found text
-				})
-				.catch(console.log)
-
-
-				//Create attachment in DB with scrape text and provided data
-				// db.promise()
-				// .query(`INSERT INTO attachment 
-				// 	(note_id, user_id, attachment_type, text, url, last_indexed, file_location) 
-				// 	VALUES (?, ?, ?, ?, ?, ?, ?)`, [noteId, userId, 1, desiredSearchText, url, created, thumbnailFilename])
-				// .then((rows, fields) => {
-
-				// 	resolve(desiredSearchText) //Return found text
-				// })
-				// .catch(console.log)
-			})
-
-		})
-		.catch(error => {
-			console.log('Scrape pooped out')
-			console.log('Issue with scrape', error.statusCode)
-			clearTimeout(requestTimeout)
-			return resolve('No site text')
-		})
-
-		requestTimeout = setTimeout( () => {
-			console.log('Cancel the request, its taking to long.')
-			request.cancel()
-			return resolve('Request Timeout')
-		}, scrapeTime )
 	})
 }
 
+const cleanupFiles = (paths) => {
+	return Promise.all(paths.map(target =>
+		fs.promises.unlink(target).catch(() => null)
+	))
+}
+
+//
+// Sanity-check the bytes against the declared content type. A server can claim
+// image/png and send anything.
+//
+const IMAGE_MAGIC = [
+	{ mime: 'image/png', bytes: [0x89, 0x50, 0x4e, 0x47] },
+	{ mime: 'image/jpeg', bytes: [0xff, 0xd8, 0xff] },
+	{ mime: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] },
+	{ mime: 'image/bmp', bytes: [0x42, 0x4d] },
+]
+
+const magicBytesMatch = (buffer, mime) => {
+	const expected = IMAGE_MAGIC.find(entry => entry.mime === mime)
+
+	// webp/avif/ico and friends have container-dependent headers - accept them
+	// on the content-type alone rather than guessing wrong.
+	if (!expected) { return true }
+	if (buffer.length < expected.bytes.length) { return false }
+
+	return expected.bytes.every((byte, index) => buffer[index] === byte)
+}
+
 Attachment.generatePushKey = (userId) => {
 	return new Promise((resolve, reject) => {
 
@@ -576,37 +562,60 @@ Attachment.getPushkeyBookmarklet = (userId) => {
 	})
 }
 
-Attachment.pushUrl = (pushkey,url) => {
+//
-	return new Promise((resolve, reject) => {
+// Bookmarklet entry point: GET /api/public/pushmebaby?pushkey=...&url=...
+//
+// The URL arrives straight off a query string, so it is validated here - the
+// old code passed it through completely unvalidated, bypassing even the https
+// normalisation that note text goes through.
+//
+// Response timing is preserved: the bookmarklet still answers after the scrape,
+// but now with a hard cap instead of the previous unbounded hang.
+//
+Attachment.pushUrl = (pushkey, url) => {
 
-		let userId = null
+	const ScrapeQueue = require('@workers/ScrapeQueue')
-		pushkey = pushkey.replace(/ /g, '+')
+	const Schema = require('@helpers/Schema')
 
-		db.promise()
+	// Preserve the bookmarklet contract - resolve truthy, leaking nothing -
+	// rather than failing an insert against columns that do not exist.
+	if (!Schema.isScrapeSchemaReady()) { return Promise.resolve(true) }
+
+	let userId = null
+	pushkey = String(pushkey || '').replace(/ /g, '+')
+
+	return db.promise()
 		.query("SELECT id FROM user WHERE pushkey = ? LIMIT 1", [pushkey])
-		.then((rows, fields) => {
+		.then(([rows]) => {
 
-			if(rows[0].length == 0){
+			// Unchanged: an unknown pushkey resolves truthy and leaks nothing
-				return resolve(true)
+			// about whether the key exists.
-			}
+			if (rows.length == 0) { return null }
 
-			userId = rows[0][0].id
+			userId = rows[0].id
-			return Attachment.scrapeUrlsCreateAttachments(userId, null, [url])			
+
+			return UrlSafety.assertFetchable(url)
+				.then(() => Attachment.createPendingUrlAttachment(userId, null, url))
+				.then(attachmentId => ScrapeQueue.awaitUrl(
+					ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url }),
+					config.totalBudgetMs
+				))
+				.catch(error => {
+					console.log('[scrape] pushUrl refused', url, error.reason || error.message)
+					return null
+				})
 		})
 		.then(() => {
-
+			if (userId) {
-			if(typeof SocketIo != 'undefined'){
+				Realtime.emitToUser(userId, 'update_counts')
-				//Once everything is done being scraped, emit new attachment events
+				Realtime.emitToUser(userId, 'update_note_attachments')
-				SocketIo.to(userId).emit('update_counts')
-
-				// Tell user to update attachments with scraped text
-				SocketIo.to(userId).emit('update_note_attachments')
 			}
-
+			return true
-			return resolve(true)
+		})
+		.catch(error => {
+			console.log('[scrape] pushUrl failed', error.message)
+			return true
 		})
-		.catch(console.log)
-	})
 }
 
 Attachment.generateBookmarkletText = (pushKey) => {

server/models/CycleTracking.js

@@ -0,0 +1,66 @@
+let db = require('@config/database')
+
+let Note = require('@models/Note')
+
+let CycleTracking = module.exports = {}
+
+
+CycleTracking.get = (userId, masterKey) => {
+	return new Promise((resolve, reject) => {
+
+		db.promise()
+		.query(`
+			SELECT note.id FROM note WHERE quick_note = 2 AND user_id = ? LIMIT 1`, [userId])
+		.then((rows, fields) => {
+
+			//Quick Note is set, return note object
+			if(rows[0][0] != undefined){
+
+				let noteId = rows[0][0].id
+				const note = Note.get(userId, noteId, masterKey)
+				.then(noteData => {
+					return resolve(noteData)
+				})
+
+			} else {
+				//Or create a new note
+				let finalId = null
+				return Note.create(userId, 'Cycle Tracking', '', masterKey)
+				.then(insertedId => {
+					finalId = insertedId
+					db.promise().query('UPDATE note SET quick_note = 2 WHERE id = ? AND user_id = ?',[insertedId, userId])
+					.then((rows, fields) => {
+
+						const note = Note.get(userId, finalId, masterKey)
+						.then(noteData => {
+							return resolve(noteData)
+						})
+
+					})
+				})
+				
+			}
+
+			
+		})
+		.catch(console.log)
+	})
+}
+
+CycleTracking.save = (userId, cycleData, masterKey) => {
+	return new Promise((resolve, reject) => {
+
+		let finalId = null
+
+		CycleTracking.get(userId, masterKey)
+		.then(noteObject => {
+
+			return Note.update(userId, noteObject.id, cycleData, noteObject.title, noteObject.color, noteObject.pinned, noteObject.archived, null, masterKey)
+			
+		})
+		.then( saveResults => {
+			return resolve(saveResults)
+		})
+	})
+
+}

server/models/Note.js

@@ -4,6 +4,7 @@ let Note = module.exports = {}
 
 let Tags = require('@models/Tag')
 let Attachment = require('@models/Attachment')
+let Realtime = require('@helpers/Realtime')
 let ShareNote = require('@models/ShareNote')
 
 let ProcessText = require('@helpers/ProcessText')
@@ -12,7 +13,6 @@ const DiffMatchPatch = require('@helpers/DiffMatchPatch')
 
 const crypto = require('crypto')
 const cs = require('@helpers/CryptoString')
-const rp = require('request-promise');
 const fs = require('fs')
 const gm = require('gm')
 
@@ -199,7 +199,7 @@ Note.create = (userId, noteTitle = '', noteText = '', masterKey) => {
 		.then((rows, fields) => {
 
 			if(typeof SocketIo != 'undefined'){
-				SocketIo.to(userId).emit('new_note_created', rows[0].insertId)
+				SocketIo.to(`user:${userId}`).emit('new_note_created', rows[0].insertId)
 			}
 
 			// Indexing is done on save
@@ -490,7 +490,7 @@ Note.update = (userId, noteId, noteText, noteTitle, color, pinned, archived, has
 					}
 					db.promise().query('UPDATE note SET snippet = ? WHERE id = ?', [updatedSnippet, otherNote.id])
 					.then((rows, fields) => {
-						SocketIo.to(otherNote['user_id']).emit('new_note_text_saved', {'noteId':otherNote.id, hash})
+						SocketIo.to(`user:${otherNote['user_id']}`).emit('new_note_text_saved', {'noteId':otherNote.id, hash})
 					})
 				}
 			})
@@ -513,12 +513,15 @@ Note.update = (userId, noteId, noteText, noteTitle, color, pinned, archived, has
 		})
 		.then((rows, fields) => {
 
-			if(typeof SocketIo != 'undefined'){
+			// The old `if(typeof SocketIo != 'undefined')` guard did not just
-				SocketIo.to(userId).emit('new_note_text_saved', {noteId, hash})
+			// skip the emit - it skipped the scrape too, so link scraping was
+			// silently disabled under Jest and could never be tested. Realtime
+			// no-ops safely without a socket server, so both now always run.
+			Realtime.emitToUser(userId, 'new_note_text_saved', {noteId, hash})
 
-				//Async attachment reindex
+			//Async attachment reindex
-				Attachment.scanTextForWebsites(SocketIo, userId, noteId, noteText)
+			Attachment.scanTextForWebsites(userId, noteId, noteText)
-			}
+				.catch(error => console.log('scanTextForWebsites failed', error.message))
 			
 			//Send back updated response
 			resolve(rows[0])
@@ -538,7 +541,7 @@ Note.setPinned = (userId, noteId, pinnedBoolean) => {
 		.query('UPDATE note JOIN note_raw_text ON note_raw_text_id = note_raw_text.id SET pinned = ?, updated = ? WHERE note.id = ? AND user_id = ?', 
 		[pinned, now, noteId, userId])
 		.then((rows, fields) => {
-			SocketIo.to(userId).emit('note_attribute_modified', noteId)
+			SocketIo.to(`user:${userId}`).emit('note_attribute_modified', noteId)
 			resolve(true)
 		})
 	})
@@ -555,7 +558,7 @@ Note.setArchived = (userId, noteId, archivedBoolead) => {
 		.query('UPDATE note JOIN note_raw_text ON note_raw_text_id = note_raw_text.id SET archived = ?, updated = ? WHERE note.id = ? AND user_id = ?', 
 		[archived, now, noteId, userId])
 		.then((rows, fields) => {
-			SocketIo.to(userId).emit('note_attribute_modified', noteId)
+			SocketIo.to(`user:${userId}`).emit('note_attribute_modified', noteId)
 			resolve(true)
 		})
 	})
@@ -581,7 +584,7 @@ Note.setTrashed = (userId, noteId, trashedBoolean, masterKey) => {
 			Note.reindex(userId, masterKey, removeFromIndex)
 			
 
-			SocketIo.to(userId).emit('note_attribute_modified', noteId)
+			SocketIo.to(`user:${userId}`).emit('note_attribute_modified', noteId)
 			resolve(true)
 		})
 	})
@@ -653,7 +656,7 @@ Note.delete = (userId, noteId, masterKey = null) => {
 				}
 			})
 
-			SocketIo.to(userId).emit('update_counts')
+			SocketIo.to(`user:${userId}`).emit('update_counts')
 
 			if(masterKey){
 				//Remove note ID from index
@@ -1191,3 +1194,29 @@ Note.search = (userId, searchQuery, searchTags, fastFilters, masterKey) => {
 		
 	})
 }
+
+//
+// Resolve a note's raw-text id for a given user.
+// Doubles as the access check: a user with no claim on this note gets null,
+// so authorisation and room derivation are the same lookup.
+//
+Note.getRawTextIdForUser = (noteId, userId) => {
+	return new Promise((resolve, reject) => {
+
+		const id = parseInt(noteId, 10)
+		if(!Number.isInteger(id) || id < 1){ return resolve(null) }
+
+		db.promise()
+		.query('SELECT note_raw_text_id FROM note WHERE id = ? AND user_id = ? LIMIT 1', [id, userId])
+		.then((rows) => {
+
+			if(!rows[0] || rows[0].length == 0){ return resolve(null) }
+
+			return resolve( rows[0][0]['note_raw_text_id'] )
+		})
+		.catch(error => {
+			console.log(error)
+			return resolve(null)
+		})
+	})
+}

server/models/ScrapeCache.js

@@ -0,0 +1,224 @@
+//
+// Read-through cache for scraped URL metadata, keyed by a hash of the
+// normalised URL and shared across users and notes.
+//
+// A fresh hit means zero network I/O. A stale hit means a conditional request
+// (If-None-Match / If-Modified-Since), so an unchanged page costs one 304
+// rather than a full page load. Failures are backed off so a dead link is not
+// refetched on every note save.
+//
+// Stores only public page metadata. Note content never reaches this table.
+//
+
+const db = require('@config/database')
+const UrlSafety = require('@helpers/UrlSafety')
+
+let ScrapeCache = module.exports = {}
+
+const SUCCESS_TTL_SECONDS = 30 * 24 * 60 * 60 // 30 days
+
+// Widening backoff on repeated failure, then give up and stay failed.
+const FAILURE_BACKOFF_SECONDS = [60 * 60, 6 * 60 * 60, 24 * 60 * 60]
+const MAX_FAILURES = FAILURE_BACKOFF_SECONDS.length
+
+ScrapeCache.SUCCESS_TTL_SECONDS = SUCCESS_TTL_SECONDS
+ScrapeCache.MAX_FAILURES = MAX_FAILURES
+
+const now = () => Math.round(Date.now() / 1000)
+
+const truncate = (value, length) => {
+	if (value == null) { return null }
+	const text = String(value)
+	return text.length > length ? text.substring(0, length) : text
+}
+
+ScrapeCache.get = async (urlOrKey) => {
+	const key = /^[0-9a-f]{64}$/.test(urlOrKey) ? urlOrKey : UrlSafety.urlKey(urlOrKey)
+	if (!key) { return null }
+
+	const [rows] = await db.promise().query('SELECT * FROM scrape_cache WHERE url_hash = ? LIMIT 1', [key])
+	return rows.length ? rows[0] : null
+}
+
+// Fresh = inside its TTL and successfully fetched.
+ScrapeCache.isFresh = (row, at) => {
+	if (!row || !row.expires_at) { return false }
+	return row.expires_at > (at || now())
+}
+
+ScrapeCache.isUsable = (row) => {
+	return !!(row && row.fetch_state === 'ok' && row.search_text != null)
+}
+
+// Has this URL failed enough times that we should stop trying?
+ScrapeCache.isExhausted = (row) => {
+	return !!(row && row.fetch_state === 'failed' && row.failure_count >= MAX_FAILURES)
+}
+
+//
+// Conditional-request headers for a stale-but-successful row. Null when there
+// is nothing to revalidate against.
+//
+ScrapeCache.conditionalFor = (row) => {
+	if (!row || row.fetch_state !== 'ok') { return null }
+	if (!row.etag && !row.last_modified) { return null }
+
+	return { etag: row.etag || null, lastModified: row.last_modified || null }
+}
+
+//
+// Reserve a row before fetching, so a concurrent save for the same URL finds
+// it rather than starting a second fetch.
+//
+ScrapeCache.markPending = async (url) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return null }
+
+	await db.promise().query(
+		`INSERT INTO scrape_cache (url_hash, url, fetch_state, fetched_at)
+		 VALUES (?, ?, 'pending', ?)
+		 ON DUPLICATE KEY UPDATE fetch_state = IF(fetch_state = 'ok', fetch_state, 'pending')`,
+		[key, truncate(url, 1000), now()]
+	)
+
+	return key
+}
+
+//
+// Store a successful scrape.
+//
+ScrapeCache.putSuccess = async (url, result) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return null }
+
+	const at = now()
+	const fields = {
+		url: truncate(url, 1000),
+		canonical_url: truncate(result.canonicalUrl, 1000),
+		status: result.status || 200,
+		title: truncate(result.title, 512),
+		site_name: truncate(result.siteName, 255),
+		description: result.description || null,
+		keywords: truncate(result.keywords, 512),
+		search_text: result.searchText || null,
+		image_url: truncate(result.imageUrl, 1000),
+		file_location: truncate(result.fileLocation, 500),
+		etag: truncate(result.etag, 255),
+		last_modified: truncate(result.lastModified, 128),
+		content_type: truncate(result.contentType, 128),
+		fetch_state: 'ok',
+		failure_count: 0,
+		last_error: null,
+		fetched_at: at,
+		expires_at: at + SUCCESS_TTL_SECONDS,
+	}
+
+	const columns = Object.keys(fields)
+	const values = columns.map(column => fields[column])
+
+	await db.promise().query(
+		`INSERT INTO scrape_cache (url_hash, ${columns.join(', ')})
+		 VALUES (?, ${columns.map(() => '?').join(', ')})
+		 ON DUPLICATE KEY UPDATE ${columns.map(column => `${column} = VALUES(${column})`).join(', ')}`,
+		[key, ...values]
+	)
+
+	return key
+}
+
+//
+// A 304. The page is unchanged, so only the freshness window moves.
+//
+// Deliberately does NOT touch attachment.last_indexed - Attachment.search
+// orders by it, so revalidating would silently reshuffle the user's list.
+//
+ScrapeCache.touchNotModified = async (url) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return null }
+
+	const at = now()
+	await db.promise().query(
+		`UPDATE scrape_cache
+		 SET fetched_at = ?, expires_at = ?, fetch_state = 'ok', failure_count = 0, last_error = NULL
+		 WHERE url_hash = ?`,
+		[at, at + SUCCESS_TTL_SECONDS, key]
+	)
+
+	return key
+}
+
+//
+// Record a failure with a widening retry window.
+//
+// `blocked` (robots or the safety gate) is terminal: retrying will not change
+// the answer, so it is parked for the full success TTL instead of backing off.
+//
+ScrapeCache.recordFailure = async (url, error) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return null }
+
+	const existing = await ScrapeCache.get(key)
+	const failureCount = (existing ? existing.failure_count : 0) + 1
+
+	const kind = (error && error.kind) || 'unknown'
+	const terminal = kind === 'blocked' || kind === 'robots_disallowed'
+
+	let ttl = null
+	if (terminal) {
+		ttl = SUCCESS_TTL_SECONDS
+	} else if (error && error.retryAfterMs) {
+		ttl = Math.round(error.retryAfterMs / 1000)
+	} else {
+		ttl = FAILURE_BACKOFF_SECONDS[Math.min(failureCount, MAX_FAILURES) - 1]
+	}
+
+	const at = now()
+	const state = terminal ? 'blocked' : 'failed'
+
+	await db.promise().query(
+		`INSERT INTO scrape_cache
+			(url_hash, url, status, fetch_state, failure_count, last_error, fetched_at, expires_at)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		 ON DUPLICATE KEY UPDATE
+			status = VALUES(status),
+			fetch_state = VALUES(fetch_state),
+			failure_count = VALUES(failure_count),
+			last_error = VALUES(last_error),
+			fetched_at = VALUES(fetched_at),
+			expires_at = VALUES(expires_at)`,
+		[
+			key,
+			truncate(url, 1000),
+			(error && error.status) || 0,
+			state,
+			failureCount,
+			truncate(`${kind}: ${(error && error.message) || ''}`, 255),
+			at,
+			at + ttl,
+		]
+	)
+
+	return key
+}
+
+//
+// Project a cache row onto the shape Attachment rows want.
+//
+ScrapeCache.toAttachmentFields = (row) => {
+	if (!row) { return null }
+	return {
+		text: row.search_text || '',
+		fileLocation: row.file_location || null,
+		url: row.canonical_url || row.url,
+	}
+}
+
+ScrapeCache.purgeExpired = async (limit = 500) => {
+	const [result] = await db.promise().query(
+		`DELETE FROM scrape_cache
+		 WHERE expires_at IS NOT NULL AND expires_at < ? AND fetch_state <> 'ok'
+		 LIMIT ?`,
+		[now(), limit]
+	)
+	return result.affectedRows || 0
+}

server/models/ShareNote.js

@@ -75,7 +75,7 @@ ShareNote.addUserToSharedNote = (userId, noteId, shareUserId, masterKey) => {
 			const sharedUserNoteId = rows[0]['insertId']
 
 			//Emit update count event to user shared with - so they see the note in real time
-			SocketIo.to(shareUserId).emit('update_counts')
+			SocketIo.to(`user:${shareUserId}`).emit('update_counts')
 
 			let success = true
 			return resolve({success, shareUserId, sharedUserNoteId})

server/models/Tag.js

@@ -58,7 +58,7 @@ Tag.addToNote = (userId, noteId, tagText) => {
 					Tag.associateWithNote(userId, noteId, newTagId)
 					.then( result => {
 
-						SocketIo.to(userId).emit('new_note_text_saved', {noteId})
+						SocketIo.to(`user:${userId}`).emit('new_note_text_saved', {noteId})
 						resolve(result)
 					})
 				})

server/routes/attachmentController.js

@@ -7,8 +7,6 @@ let router = express.Router()
 let Attachment = require('@models/Attachment')
 let Note = require('@models/Note')
 
-let userId = null
-let masterKey = null
 
 // middleware that is specific to this router
 router.use(function setUserId (req, res, next) {
@@ -19,23 +17,33 @@ router.use(function setUserId (req, res, next) {
 	}
 
 	if(req.headers.userId){
-		userId = req.headers.userId
-		masterKey = req.headers.masterKey
 		next()
 	}
 })
 
 router.post('/search', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Attachment.search(userId, req.body.noteId, req.body.attachmentType, req.body.offset, req.body.setSize, req.body.includeShared)
 	.then( data => res.send(data) )
 })
 
 router.post('/textsearch', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Attachment.textSearch(userId, req.body.searchTerm)
 	.then( data => res.send(data) )
 })
 
 router.post('/update', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Attachment.update(userId, req.body.attachmentId, req.body.updatedText, req.body.noteId)
 	.then( result => {
 		Note.reindex(userId, req.body.noteId)
@@ -44,11 +52,19 @@ router.post('/update', function (req, res) {
 })
 
 router.post('/delete', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Attachment.delete(userId, req.body.attachmentId)
 	.then( data => res.send(data) )
 })
 
 router.post('/upload', upload.single('file'), function (req, res, next) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 
 	//Create attachment with file information and node id
 	const noteId = parseInt(req.body.noteId)
@@ -67,6 +83,10 @@ router.post('/upload', upload.single('file'), function (req, res, next) {
 
 // get push key
 router.post('/getbookmarklet', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 
 	Attachment.getPushkeyBookmarklet(userId)
 	.then( data => res.send(data) )

server/routes/cycletrackingController.js

@@ -0,0 +1,45 @@
+//
+// /api/cycle-tracking
+//
+
+var express = require('express')
+var router = express.Router()
+
+let CycleTracking = require('@models/CycleTracking');
+
+
+// middleware that is specific to this router
+router.use(function setUserId (req, res, next) {
+
+	//Session key is required to continue
+	if(!req.headers.sessionId){
+		next('Unauthorized')
+	}
+
+	if(req.headers.userId){
+		next()
+	}
+})
+
+//Get quick note text
+router.post('/get', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
+	CycleTracking.get(userId, masterKey)
+	.then( data => res.send(data) )
+})
+
+//Push text to quick note
+router.post('/save', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
+	CycleTracking.save(userId, req.body.cycleData, masterKey)
+	.then( data => res.send(data) )
+})
+
+
+module.exports = router

server/routes/metrictrackingController.js

@@ -7,8 +7,6 @@ var router = express.Router()
 
 let MetricTracking = require('@models/MetricTracking');
 
-let userId = null
-let masterKey = null
 
 // middleware that is specific to this router
 router.use(function setUserId (req, res, next) {
@@ -19,24 +17,34 @@ router.use(function setUserId (req, res, next) {
 	}
 
 	if(req.headers.userId){
-		userId = req.headers.userId
-		masterKey = req.headers.masterKey
 		next()
 	}
 })
 
 router.post('/get', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	MetricTracking.get(userId, masterKey)
 	.then( data => res.send(data) )
 })
 
 router.post('/create', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	MetricTracking.create(userId, masterKey)
 	.then( data => res.send(data) )
 })
 
 //Push text to quick note
 router.post('/save', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	MetricTracking.save(userId, req.body.cycleData, masterKey)
 	.then( data => res.send(data) )
 })

server/routes/noteController.js

@@ -5,8 +5,6 @@ let Note = require('@models/Note')
 let User = require('@models/User')
 let ShareNote = require('@models/ShareNote')
 
-let userId = null
-let masterKey = null
 
 // middleware that is specific to this router
 router.use(function setUserId (req, res, next) {
@@ -17,8 +15,6 @@ router.use(function setUserId (req, res, next) {
 	}
 
 	if(req.headers.userId){
-		userId = req.headers.userId
-		masterKey = req.headers.masterKey
 		next()
 	}
 })
@@ -27,6 +23,10 @@ router.use(function setUserId (req, res, next) {
 // Note actions
 //
 router.post('/get', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.get(userId, req.body.noteId, masterKey)
 	.then( noteObject => {
 
@@ -39,21 +39,37 @@ router.post('/get', function (req, res) {
 })
 
 router.post('/delete', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.delete(userId, req.body.noteId, masterKey)
 	.then( data => res.send(data) )
 })
 
 router.post('/create', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.create(userId, req.body.title, req.body.text, masterKey)
 	.then( id => res.send({id}) )
 })
 
 router.post('/update', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.update(userId, req.body.noteId, req.body.text, req.body.title, req.body.color, req.body.pinned, req.body.archived, req.body.hash, masterKey)
 	.then( id => res.send({id}) )
 })
 
 router.post('/search', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.search(userId, req.body.searchQuery, req.body.searchTags, req.body.fastFilters, masterKey)
 	.then( NoteAndTags => {
 		res.send(NoteAndTags)
@@ -61,6 +77,10 @@ router.post('/search', function (req, res) {
 })
 
 router.post('/reindex', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.reindex(userId, masterKey)
 	.then( data => {
 		res.send(data)
@@ -72,18 +92,30 @@ router.post('/reindex', function (req, res) {
 // Update single note attributes
 //
 router.post('/setpinned', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.setPinned(userId, req.body.noteId, req.body.pinned)
 	.then( results => {
 		res.send(results)
 	})
 })
 router.post('/setarchived', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.setArchived(userId, req.body.noteId, req.body.archived)
 	.then( results => {
 		res.send(results)
 	})
 })
 router.post('/settrashed', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Note.setTrashed(userId, req.body.noteId, req.body.trashed, masterKey)
 	.then( results => {
 		res.send(results)
@@ -94,11 +126,19 @@ router.post('/settrashed', function (req, res) {
 // Share Note Actions
 //
 router.post('/getshareinfo', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	ShareNote.getShareInfo(userId, req.body.noteId, req.body.rawTextId)
 	.then(results => res.send(results))
 })
 
 router.post('/shareadduser', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	// ShareNote.addUser(userId, req.body.noteId, req.body.rawTextId, req.body.username, masterKey)
 	User.getByUserName(req.body.username)
 	.then( user => {
@@ -111,22 +151,38 @@ router.post('/shareadduser', function (req, res) {
 })
 
 router.post('/shareremoveuser', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	// (userId, noteId, shareNoteUserId, shareUserId, masterKey)
 	ShareNote.removeUserFromSharedNote(userId, req.body.noteId, req.body.shareUserNoteId, masterKey)
 	.then(results => res.send(results))
 })
 
 router.post('/enableshare', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	//Create Shared Encryption Key for Note
 	ShareNote.migrateToShared(userId, req.body.noteId, masterKey)
 	.then(results => res.send(true))
 })
 router.post('/getsharekey', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	//Get Shared Key for a note
 	ShareNote.decryptSharedKey(userId, req.body.noteId, masterKey)
 	.then(results => res.send(results))
 })
 router.post('/disableshare', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	//Removed shared encryption key from note
 	ShareNote.migrateToNormal(userId, req.body.noteId, masterKey)
 	.then(results => res.send(true))

server/routes/quicknoteController.js

@@ -3,8 +3,6 @@ var router = express.Router()
 
 let QuickNote = require('@models/QuickNote');
 
-let userId = null
-let masterKey = null
 
 // middleware that is specific to this router
 router.use(function setUserId (req, res, next) {
@@ -15,26 +13,36 @@ router.use(function setUserId (req, res, next) {
 	}
 
 	if(req.headers.userId){
-		userId = req.headers.userId
-		masterKey = req.headers.masterKey
 		next()
 	}
 })
 
 //Get quick note text
 router.post('/get', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	QuickNote.get(userId, masterKey)
 	.then( data => res.send(data) )
 })
 
 //Push text to quick note
 router.post('/update', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	QuickNote.update(userId, req.body.pushText, masterKey)
 	.then( data => res.send(data) )
 })
 
 //Push text to quick note
 router.post('/new', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	QuickNote.newNote(userId)
 	.then( data => res.send(data) )
 })

server/routes/tagController.js

@@ -3,8 +3,6 @@ var router = express.Router()
 
 let Tags = require('@models/Tag')
 
-let userId = null
-let masterKey = null
 
 // middleware that is specific to this router
 router.use(function setUserId (req, res, next) {
@@ -15,49 +13,75 @@ router.use(function setUserId (req, res, next) {
 	}
 
 	if(req.headers.userId){
-		userId = req.headers.userId
-		masterKey = req.headers.masterKey
 		next()
 	}
 })
 
 //Get the latest notes the user has created
 router.post('/suggest', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Tags.suggest(userId, req.body.noteId, req.body.tagText)
 	.then( data => res.send(data) )
 })
 
 //Get latest tags based on when it was put on a note
 router.post('/latest', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Tags.latest(userId, req.body.noteId)
 	.then( data => res.send(data) )
 })
 
 //Get the latest notes the user has created
 router.post('/addtonote', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Tags.addToNote(userId, req.body.noteId, req.body.tagText.toLowerCase())
 	.then( data => res.send(data) )
 })
 
 router.post('/removefromnote', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Tags.removeTagFromNote(userId, req.body.tagId)
 	.then( data => res.send(data) )
 })
 
 //Get the latest notes the user has created
 router.post('/get', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Tags.get(userId, req.body.noteId)
 	.then( data => res.send(data) )
 })
 
 //Get the latest notes the user has created
 router.post('/fornote', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Tags.fornote(userId, req.body.noteId)
 	.then( data => res.send(data) )
 })
 
 //Get all the tags for this user in order of usage
 router.post('/usertags', function (req, res) {
+	//Bind per request. These were module-level singletons shared by every
+	//concurrent request, so one user's masterKey could overwrite another's.
+	const { userId, masterKey } = req.headers
+
 	Tags.userTags(userId, req.body.searchQuery, req.body.searchTags, req.body.fastFilters)
 	.then( data => res.send(data) )
 })

server/routes/userController.js

@@ -5,8 +5,6 @@ const User = require('@models/User')
 const Auth = require('@helpers/Auth')
 const cs = require('@helpers/CryptoString')
 
-let userId = null
-let masterKey = null
 
 // middleware that is specific to this router
 router.use(function setUserId (req, res, next) {
@@ -17,8 +15,6 @@ router.use(function setUserId (req, res, next) {
 	}
 
 	if(req.headers.userId){
-		userId = req.headers.userId
-		masterKey = req.headers.masterKey
 		next()
 	}
 })

server/tests/fixtures/html/README.md

@@ -0,0 +1,7 @@
+# HTML fixtures
+
+Frozen, hand-trimmed snapshots used by `server/tests/helpers/SiteScrape.test.js`.
+
+These are **not** fetched live. Each file isolates one parsing situation so a
+failure points at a specific branch. Add a new file rather than growing an
+existing one when covering a new case.

server/tests/fixtures/html/apple-touch-icon.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head>
+  <title>Icons By Size</title>
+  <link rel="icon" href="https://example.com/small.png" sizes="16x16">
+  <link rel="apple-touch-icon" href="https://example.com/touch-180.png" sizes="180x180">
+</head>
+<body><p>x</p></body>
+</html>