From 9804771a5b8fa912852704b7463445c3a8fcd17b Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 3 Nov 2026 12:53:17 -0700 Subject: [PATCH] Testing Migrations --- migrateDatabase.sh | 225 +++++++++++++++++++++++ prisma.config.mjs | 32 ++++ prisma/migrations/0_init/migration.sql | 211 +++++++++++++++++++++ prisma/migrations/migration_lock.toml | 3 + prisma/schema.prisma | 151 +++++++++++++++ prisma/seed.sql | 24 +++ server/tests/helpers/Schema.test.js | 243 +++++++++++++++++++++++++ server/tests/schema/migrations.test.js | 220 ++++++++++++++++++++++ 8 files changed, 1109 insertions(+) create mode 100755 migrateDatabase.sh create mode 100644 prisma.config.mjs create mode 100644 prisma/migrations/0_init/migration.sql create mode 100644 prisma/migrations/migration_lock.toml create mode 100644 prisma/schema.prisma create mode 100644 prisma/seed.sql create mode 100644 server/tests/helpers/Schema.test.js create mode 100644 server/tests/schema/migrations.test.js diff --git a/migrateDatabase.sh b/migrateDatabase.sh new file mode 100755 index 0000000..c7f9c71 --- /dev/null +++ b/migrateDatabase.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# +# Apply database migrations. Run from the repo root, INSIDE nix-shell (that is +# where the prisma CLI lives). +# +# nix-shell shell.nix --run ./migrateDatabase.sh +# +# Works both ways round: it applies pending migrations to an existing database, +# and builds a complete one from empty. Safe to re-run -- a second run is a +# no-op. Replaces migrateDatabase.sh. +# +# --- why this is a separate step ----------------------------------------- +# +# The application connects as a least-privilege account (production uses `app`) +# holding DML rights only -- no CREATE, ALTER or DROP. That is deliberate: the +# server fetches untrusted URLs for link previews, and a service able to reshape +# its own tables is a far larger blast radius than that feature warrants. +# +# So the app never applies DDL. It verifies the schema at boot +# (server/helpers/Schema.js) and disables link previews if anything is missing. +# +# --- authoring a new migration ------------------------------------------- +# +# NEVER run `prisma migrate dev`. It can decide the database has drifted and +# offer to RESET it, which on a box holding a production data restore would +# destroy real data. It also wants a shadow database. Instead: +# +# 1. Edit prisma/schema.prisma +# 2. NAME=add_whatever +# DIR="prisma/migrations/$(date +%Y%m%d%H%M%S)_$NAME" +# mkdir -p "$DIR" +# prisma migrate diff --from-config-datasource \ +# --to-schema prisma/schema.prisma --script > "$DIR/migration.sql" +# 3. READ the generated SQL -- this review is what makes the approach safe +# 4. ./migrateDatabase.sh +# +# Note: Prisma emits `COLLATE utf8mb4_unicode_ci`, but this database is +# `utf8mb4_general_ci` throughout, and `prisma migrate diff` does NOT report the +# difference. Fix the collation in any generated migration by hand. This script +# checks for drift at the end. +# +# --- credentials --------------------------------------------------------- +# +# Same convention as applyLastBackup.sh / backupDatabase.sh: source ~/.env and +# connect as root with $PROD_DB_PASS. Falls back to passwordless root (the dev +# case, where shell.nix also exports DATABASE_URL). Override with: +# +# DB_ADMIN_USER=admin DB_ADMIN_PASS=secret ./migrateDatabase.sh +# +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$REPO" + +DB_NAME="application" # hardcoded in server/config/database.js too + +# --- preconditions ------------------------------------------------------- + +if ! command -v prisma >/dev/null 2>&1; then + echo "prisma is not on PATH -- run this inside nix-shell:" >&2 + echo " nix-shell shell.nix --run ./migrateDatabase.sh" >&2 + exit 1 +fi + +if ! command -v mariadb >/dev/null 2>&1 && ! command -v mysql >/dev/null 2>&1; then + echo "no mariadb/mysql client on PATH -- run this inside nix-shell." >&2 + exit 1 +fi + +if command -v mariadb >/dev/null 2>&1; then MYSQL_BIN="mariadb"; else MYSQL_BIN="mysql"; fi + +if ! command -v node >/dev/null 2>&1; then + echo "node is not on PATH -- needed to percent-encode the database password." >&2 + exit 1 +fi + +if [ -f "$HOME/.env" ]; then + # shellcheck disable=SC1091 + source "$HOME/.env" +fi + +ADMIN_USER="${DB_ADMIN_USER:-root}" +ADMIN_PASS="${DB_ADMIN_PASS:-${PROD_DB_PASS:-}}" + +mysql_args=(--user "$ADMIN_USER") +[ -n "$ADMIN_PASS" ] && mysql_args+=(--password="$ADMIN_PASS") + +LOCAL_SOCKET="$REPO/.mysql/mysql.sock" +[ -S "$LOCAL_SOCKET" ] && mysql_args+=(--socket="$LOCAL_SOCKET") + +# Prisma needs a URL, and it must carry the ADMIN credentials -- never the +# app account, which holds no DDL. +DB_ADMIN_HOST="${DB_ADMIN_HOST:-127.0.0.1}" +DB_ADMIN_PORT="${DB_ADMIN_PORT:-3306}" + +# +# Percent-encode the password. Production passwords routinely contain @ : / ? # +# or %, every one of which silently corrupts a URL -- an unencoded `@` would +# truncate the host and produce a baffling connection error rather than an +# obvious one. +# +encode_component() { + node -e 'process.stdout.write(encodeURIComponent(process.argv[1] || ""))' "$1" +} + +ADMIN_USER_ENC="$(encode_component "$ADMIN_USER")" + +if [ -n "$ADMIN_PASS" ]; then + ADMIN_PASS_ENC="$(encode_component "$ADMIN_PASS")" + export DATABASE_URL="mysql://${ADMIN_USER_ENC}:${ADMIN_PASS_ENC}@${DB_ADMIN_HOST}:${DB_ADMIN_PORT}/${DB_NAME}" +else + export DATABASE_URL="mysql://${ADMIN_USER_ENC}@${DB_ADMIN_HOST}:${DB_ADMIN_PORT}/${DB_NAME}" +fi + +# Keep the CLI from phoning checkpoint.prisma.io for update checks. +export CHECKPOINT_DISABLE=1 + +if ! "$MYSQL_BIN" "${mysql_args[@]}" -e "SELECT 1" >/dev/null 2>&1; then + echo "Cannot connect to the database as '$ADMIN_USER'." >&2 + echo "Set PROD_DB_PASS in ~/.env, or pass DB_ADMIN_USER / DB_ADMIN_PASS." >&2 + exit 1 +fi + +# +# Collation is explicit because the server default is utf8mb4_unicode_ci while +# every application table is utf8mb4_general_ci. A table created later without +# an explicit COLLATE would otherwise inherit the wrong one, changing string +# comparison and FULLTEXT ranking -- and Prisma neither models collation nor +# reports the difference in `migrate diff`. +# +# For a database that already exists with the wrong default, the one-time +# non-destructive fix is: +# ALTER DATABASE application CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +# +"$MYSQL_BIN" "${mysql_args[@]}" -e "CREATE DATABASE IF NOT EXISTS \`$DB_NAME\` \ + CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" + +sql() { "$MYSQL_BIN" "${mysql_args[@]}" -N -B "$DB_NAME" -e "$1" 2>/dev/null; } + +echo '-------' +echo "Migrating '$DB_NAME' as '$ADMIN_USER'" +echo '-------' + +# --- baseline an existing, pre-Prisma database --------------------------- +# +# This is the single most dangerous moment in adopting Prisma. A database that +# already has the schema but no _prisma_migrations table must have the baseline +# marked applied, NOT executed -- running 0_init against it would fail on +# existing tables. Detect and handle it here rather than leaving it as a manual +# step someone forgets. +# +HAS_MIGRATION_TABLE=$(sql "SELECT COUNT(*) FROM information_schema.TABLES + WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='_prisma_migrations'" || echo 0) +HAS_EXISTING_SCHEMA=$(sql "SELECT COUNT(*) FROM information_schema.TABLES + WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment'" || echo 0) + +if [ "${HAS_MIGRATION_TABLE:-0}" -eq 0 ] && [ "${HAS_EXISTING_SCHEMA:-0}" -gt 0 ]; then + echo "Existing schema found with no migration history -- baselining." + echo "Marking 0_init as already applied (NOT executing it)." + prisma migrate resolve --applied 0_init + echo "" +fi + +# --- apply --------------------------------------------------------------- + +prisma migrate deploy + +# --- seed ---------------------------------------------------------------- + +if [ -f prisma/seed.sql ]; then + echo "" + echo "Applying reference data (prisma/seed.sql)..." + "$MYSQL_BIN" "${mysql_args[@]}" "$DB_NAME" < prisma/seed.sql +fi + +# --- verify -------------------------------------------------------------- +# +# Print proof rather than leaving the operator to trust us. +# +echo "" +echo '-------' +echo 'Verification' +echo '-------' + +prisma migrate status + +echo "" +"$MYSQL_BIN" "${mysql_args[@]}" "$DB_NAME" --table -e " + SELECT 'attachment FULLTEXT index' AS check_name, + IFNULL(MAX(INDEX_TYPE), 'MISSING') AS result + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND INDEX_NAME = 'attachment_text_index' + UNION ALL + SELECT 'tables', + CAST(COUNT(*) AS CHAR) FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME <> '_prisma_migrations' + UNION ALL + SELECT 'attachment_type seed rows', + CAST(COUNT(*) AS CHAR) FROM attachment_type + UNION ALL + -- _prisma_migrations is Prisma's own bookkeeping table and is created with + -- its default collation; only application tables are checked here. + SELECT 'non-general_ci app tables (expect 0)', + CAST(COUNT(*) AS CHAR) FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME <> '_prisma_migrations' + AND TABLE_COLLATION <> 'utf8mb4_general_ci'; +" + +# The FULLTEXT index backs Attachment.textSearch. If a migration ever converts +# it to a BTREE index, attachment search silently returns nothing -- so fail +# loudly here rather than letting that ship. +FT=$(sql "SELECT IFNULL(MAX(INDEX_TYPE),'MISSING') FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA='$DB_NAME' AND INDEX_NAME='attachment_text_index'") +if [ "$FT" != "FULLTEXT" ]; then + echo "FAILED: attachment_text_index is '$FT', expected FULLTEXT." >&2 + echo "Attachment search depends on MATCH..AGAINST and is now broken." >&2 + exit 1 +fi + +echo "" +echo '-------' +echo 'Done. Restart the API so it re-verifies the schema:' +echo ' pm2 restart NoteServer' +echo '-------' diff --git a/prisma.config.mjs b/prisma.config.mjs new file mode 100644 index 0000000..c72ece5 --- /dev/null +++ b/prisma.config.mjs @@ -0,0 +1,32 @@ +// +// Prisma CLI configuration. +// +// Prisma 7 removed `url` from the datasource block in schema.prisma; the +// connection string for migrations lives here instead. +// +// This file configures the CLI only. The running server does not use Prisma -- +// it talks to MariaDB through server/config/database.js (mysql2) with raw SQL. +// Prisma is a development and admin tool here, for schema migrations and +// scaffolding. +// +// DATABASE_URL must point at an account with DDL rights. In dev that is the +// root user exported by shell.nix; in production migrateDatabase.sh composes it +// from the admin credentials in ~/.env. It must NEVER be the least-privilege +// `app` account the server runs as, which deliberately holds no CREATE/ALTER. +// +// .mjs rather than .js because this project is CommonJS and Prisma's example +// config is ESM; the explicit extension avoids any ambiguity. +// +export default { + schema: 'prisma/schema.prisma', + + migrations: { + path: 'prisma/migrations', + }, + + datasource: { + // process.env directly rather than env() -- the helper throws when the + // variable is unset, which makes the failure harder to read. + url: process.env.DATABASE_URL, + }, +} diff --git a/prisma/migrations/0_init/migration.sql b/prisma/migrations/0_init/migration.sql new file mode 100644 index 0000000..c98c9cf --- /dev/null +++ b/prisma/migrations/0_init/migration.sql @@ -0,0 +1,211 @@ +-- +-- Baseline migration: the schema as it existed when Prisma was adopted. +-- +-- Generated with: +-- prisma migrate diff --from-empty --to-schema prisma/schema.prisma --script +-- +-- HAND-CORRECTED, and it matters: Prisma renders `COLLATE utf8mb4_unicode_ci` +-- for MySQL, but every live table is `utf8mb4_general_ci`. Prisma does not +-- model collation and its own `migrate diff` does NOT report the difference, so +-- without this correction a database rebuilt from empty would silently differ +-- from production in string comparison, sorting and FULLTEXT ranking. +-- If you regenerate this file, re-apply the collation fix. There is a test for +-- it in server/tests/schema/migrations.test.js. +-- +-- Things Prisma cannot express, which therefore need handling elsewhere: +-- * the DATABASE-level default collation -- set by migrateDatabase.sh when it +-- creates the database, since tables created later without an explicit +-- COLLATE would otherwise inherit the server default +-- * ENGINE=InnoDB (relies on the server default, which is InnoDB) +-- * AUTO_INCREMENT start values (cosmetic; a fresh database starting at 1 is +-- correct) +-- * the COMMENT on note_raw_text.text +-- * row data -- see prisma/seed.sql for the attachment_type lookup rows +-- +-- Context carried over from the migration this replaces: scrape_cache.url_hash +-- carries the UNIQUE index rather than `url` because a UNIQUE on +-- varchar(1000) utf8mb4 is 4000 bytes and exceeds InnoDB's 3072-byte key limit; +-- and attachment.scrape_state DEFAULT 2 ("done") is load-bearing -- it keeps +-- pre-existing attachment rows out of the scraper's boot recovery sweep. +-- +-- On environments that already had this schema (dev and production), this file +-- is NOT executed -- it is marked applied with `prisma migrate resolve`. +-- + +-- CreateTable +CREATE TABLE `attachment` ( + `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + `note_id` INTEGER UNSIGNED NULL, + `user_id` INTEGER UNSIGNED NOT NULL, + `attachment_type` INTEGER UNSIGNED NULL, + `text` MEDIUMTEXT NULL, + `url` VARCHAR(1000) NULL DEFAULT '', + `last_indexed` INTEGER NULL, + `file_location` VARCHAR(500) NULL, + `visible` TINYINT NULL DEFAULT 1, + `scrape_state` TINYINT NOT NULL DEFAULT 2, + `scrape_attempts` TINYINT NOT NULL DEFAULT 0, + `enqueued_at` INTEGER NULL, + + INDEX `attachment_scrape_state`(`scrape_state`, `enqueued_at`), + INDEX `attachment_type`(`attachment_type`), + INDEX `note_id`(`note_id`), + INDEX `user_id`(`user_id`), + FULLTEXT INDEX `attachment_text_index`(`text`), + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `attachment_type` ( + `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + `mime_type` VARCHAR(500) NULL, + `extension` VARCHAR(10) NULL, + `name` VARCHAR(50) NULL, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `note` ( + `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + `user_id` INTEGER UNSIGNED NULL, + `note_raw_text_id` INTEGER NULL, + `opened` INTEGER NULL, + `created` INTEGER NULL, + `pinned` INTEGER NULL DEFAULT 0, + `archived` INTEGER NULL DEFAULT 0, + `trashed` TINYINT NULL DEFAULT 0, + `color` VARCHAR(500) NULL, + `shared` INTEGER NULL DEFAULT 0, + `share_user_id` INTEGER NULL, + `quick_note` INTEGER NULL DEFAULT 0, + `indexed` TINYINT NULL DEFAULT 0, + `snippet` VARCHAR(1300) NULL, + `snippet_salt` VARCHAR(1300) NULL, + `encrypted_share_password_key` VARCHAR(1300) NULL, + + INDEX `user_note`(`user_id`), + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `note_raw_text` ( + `id` INTEGER NOT NULL AUTO_INCREMENT, + `text` LONGTEXT NULL, + `updated` BIGINT NULL, + `salt` VARCHAR(1300) NULL, + `last_decrypted_date` INTEGER NULL, + `decrypt_attempts_count` INTEGER NULL DEFAULT 0, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `note_tag` ( + `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + `note_id` INTEGER UNSIGNED NULL, + `tag_id` INTEGER UNSIGNED NULL, + `user_id` INTEGER UNSIGNED NULL, + + INDEX `note_id`(`note_id`), + INDEX `tag_id`(`tag_id`), + INDEX `user_id`(`user_id`), + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `scrape_cache` ( + `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + `url_hash` CHAR(64) NOT NULL, + `url` VARCHAR(1000) NOT NULL DEFAULT '', + `canonical_url` VARCHAR(1000) NULL, + `status` SMALLINT NOT NULL DEFAULT 0, + `title` VARCHAR(512) NULL, + `site_name` VARCHAR(255) NULL, + `description` MEDIUMTEXT NULL, + `keywords` VARCHAR(512) NULL, + `search_text` MEDIUMTEXT NULL, + `image_url` VARCHAR(1000) NULL, + `file_location` VARCHAR(500) NULL, + `etag` VARCHAR(255) NULL, + `last_modified` VARCHAR(128) NULL, + `content_type` VARCHAR(128) NULL, + `fetch_state` VARCHAR(20) NOT NULL DEFAULT 'pending', + `failure_count` INTEGER NOT NULL DEFAULT 0, + `last_error` VARCHAR(255) NULL, + `fetched_at` INTEGER NULL, + `expires_at` INTEGER NULL, + + UNIQUE INDEX `scrape_cache_url_hash`(`url_hash`), + INDEX `scrape_cache_expires_at`(`expires_at`), + INDEX `scrape_cache_fetch_state`(`fetch_state`), + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `tag` ( + `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + `text` VARCHAR(400) NULL, + `hash` INTEGER NULL, + + INDEX `hash`(`hash`), + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `user` ( + `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + `username` VARCHAR(150) NOT NULL DEFAULT '', + `password` TEXT NOT NULL, + `salt` VARCHAR(500) NOT NULL DEFAULT '', + `iterations` INTEGER NOT NULL, + `last_login` DATETIME(0) NULL, + `created` DATETIME(0) NULL, + `two_fa_secret` VARCHAR(50) NULL, + `two_fa_enabled` TINYINT NULL DEFAULT 0, + `two_fa_every_login` TINYINT NULL DEFAULT 0, + `pushkey` VARCHAR(100) NULL, + + UNIQUE INDEX `id_UNIQUE`(`id`), + UNIQUE INDEX `username_UNIQUE`(`username`), + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `user_active_session` ( + `id` INTEGER NOT NULL AUTO_INCREMENT, + `salt` VARCHAR(100) NULL, + `encrypted_master_password` VARCHAR(500) NULL, + `created` INTEGER NULL, + `uses` INTEGER NULL, + `user_hash` VARCHAR(500) NULL, + `active` TINYINT NULL DEFAULT 1, + `session_id` VARCHAR(100) NULL, + `last_used` INTEGER NULL, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `user_encrypted_search_index` ( + `id` INTEGER NOT NULL AUTO_INCREMENT, + `user_id` INTEGER NULL, + `salt` VARCHAR(100) NULL, + `index` LONGTEXT NULL, + `last_update` INTEGER NULL, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- CreateTable +CREATE TABLE `user_key` ( + `id` INTEGER NOT NULL AUTO_INCREMENT, + `user_id` INTEGER NULL, + `salt` VARCHAR(1300) NULL, + `key` VARCHAR(5000) NULL, + `created` INTEGER NULL, + `public_key` VARCHAR(5000) NULL, + `private_key_encrypted` VARCHAR(5000) NULL, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..592fc0b --- /dev/null +++ b/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (e.g., Git) +provider = "mysql" diff --git a/prisma/schema.prisma b/prisma/schema.prisma new file mode 100644 index 0000000..04fabd7 --- /dev/null +++ b/prisma/schema.prisma @@ -0,0 +1,151 @@ +datasource db { + provider = "mysql" +} + +model attachment { + id Int @id @default(autoincrement()) @db.UnsignedInt + note_id Int? @db.UnsignedInt + user_id Int @db.UnsignedInt + attachment_type Int? @db.UnsignedInt + text String? @db.MediumText + url String? @default("") @db.VarChar(1000) + last_indexed Int? + file_location String? @db.VarChar(500) + visible Int? @default(1) @db.TinyInt + scrape_state Int @default(2) @db.TinyInt + scrape_attempts Int @default(0) @db.TinyInt + enqueued_at Int? + + @@index([scrape_state, enqueued_at], map: "attachment_scrape_state") + @@index([attachment_type], map: "attachment_type") + @@index([note_id], map: "note_id") + @@index([user_id], map: "user_id") + @@fulltext([text], map: "attachment_text_index") +} + +model attachment_type { + id Int @id @default(autoincrement()) @db.UnsignedInt + mime_type String? @db.VarChar(500) + extension String? @db.VarChar(10) + name String? @db.VarChar(50) +} + +model note { + id Int @id @default(autoincrement()) @db.UnsignedInt + user_id Int? @db.UnsignedInt + note_raw_text_id Int? + opened Int? + created Int? + pinned Int? @default(0) + archived Int? @default(0) + trashed Int? @default(0) @db.TinyInt + color String? @db.VarChar(500) + shared Int? @default(0) + share_user_id Int? + quick_note Int? @default(0) + indexed Int? @default(0) @db.TinyInt + snippet String? @db.VarChar(1300) + snippet_salt String? @db.VarChar(1300) + encrypted_share_password_key String? @db.VarChar(1300) + + @@index([user_id], map: "user_note") +} + +/// This model or at least one of its fields has comments in the database, and requires an additional setup for migrations: Read more: https://pris.ly/d/database-comments +model note_raw_text { + id Int @id @default(autoincrement()) + text String? @db.LongText + updated BigInt? + salt String? @db.VarChar(1300) + last_decrypted_date Int? + decrypt_attempts_count Int? @default(0) +} + +model note_tag { + id Int @id @default(autoincrement()) @db.UnsignedInt + note_id Int? @db.UnsignedInt + tag_id Int? @db.UnsignedInt + user_id Int? @db.UnsignedInt + + @@index([note_id], map: "note_id") + @@index([tag_id], map: "tag_id") + @@index([user_id], map: "user_id") +} + +model scrape_cache { + id Int @id @default(autoincrement()) @db.UnsignedInt + url_hash String @unique(map: "scrape_cache_url_hash") @db.Char(64) + url String @default("") @db.VarChar(1000) + canonical_url String? @db.VarChar(1000) + status Int @default(0) @db.SmallInt + title String? @db.VarChar(512) + site_name String? @db.VarChar(255) + description String? @db.MediumText + keywords String? @db.VarChar(512) + search_text String? @db.MediumText + image_url String? @db.VarChar(1000) + file_location String? @db.VarChar(500) + etag String? @db.VarChar(255) + last_modified String? @db.VarChar(128) + content_type String? @db.VarChar(128) + fetch_state String @default("pending") @db.VarChar(20) + failure_count Int @default(0) + last_error String? @db.VarChar(255) + fetched_at Int? + expires_at Int? + + @@index([expires_at], map: "scrape_cache_expires_at") + @@index([fetch_state], map: "scrape_cache_fetch_state") +} + +model tag { + id Int @id @default(autoincrement()) @db.UnsignedInt + text String? @db.VarChar(400) + hash Int? + + @@index([hash], map: "hash") +} + +model user { + id Int @id @unique(map: "id_UNIQUE") @default(autoincrement()) @db.UnsignedInt + username String @unique(map: "username_UNIQUE") @default("") @db.VarChar(150) + password String @db.Text + salt String @default("") @db.VarChar(500) + iterations Int + last_login DateTime? @db.DateTime(0) + created DateTime? @db.DateTime(0) + two_fa_secret String? @db.VarChar(50) + two_fa_enabled Int? @default(0) @db.TinyInt + two_fa_every_login Int? @default(0) @db.TinyInt + pushkey String? @db.VarChar(100) +} + +model user_active_session { + id Int @id @default(autoincrement()) + salt String? @db.VarChar(100) + encrypted_master_password String? @db.VarChar(500) + created Int? + uses Int? + user_hash String? @db.VarChar(500) + active Int? @default(1) @db.TinyInt + session_id String? @db.VarChar(100) + last_used Int? +} + +model user_encrypted_search_index { + id Int @id @default(autoincrement()) + user_id Int? + salt String? @db.VarChar(100) + index String? @db.LongText + last_update Int? +} + +model user_key { + id Int @id @default(autoincrement()) + user_id Int? + salt String? @db.VarChar(1300) + key String? @db.VarChar(5000) + created Int? + public_key String? @db.VarChar(5000) + private_key_encrypted String? @db.VarChar(5000) +} diff --git a/prisma/seed.sql b/prisma/seed.sql new file mode 100644 index 0000000..a7d9772 --- /dev/null +++ b/prisma/seed.sql @@ -0,0 +1,24 @@ +-- +-- Reference data required for a working database. +-- +-- Applied by ./migrateDatabase.sh after migrations. Kept out of the migration +-- history on purpose: environments that were baselined with +-- `prisma migrate resolve` never execute 0_init, so seed rows placed inside it +-- would silently never exist there. +-- +-- Idempotent, so re-running is harmless. +-- + +-- attachment_type is a lookup table the application joins against. +-- attachment_type = 1 is hardcoded for scraped URLs, 2 for uploaded images, +-- 3 for PDFs (see server/models/Attachment.js). Without these rows a freshly +-- scaffolded database creates attachments that reference nothing. +INSERT INTO `attachment_type` (`id`, `mime_type`, `extension`, `name`) VALUES + (1, NULL, NULL, 'URL'), + (2, NULL, '.jpg', 'Image - Jpg'), + (3, NULL, '.pdf', 'PDF'), + (4, NULL, '.doc', 'DOC') +ON DUPLICATE KEY UPDATE + `mime_type` = VALUES(`mime_type`), + `extension` = VALUES(`extension`), + `name` = VALUES(`name`); diff --git a/server/tests/helpers/Schema.test.js b/server/tests/helpers/Schema.test.js new file mode 100644 index 0000000..8296303 --- /dev/null +++ b/server/tests/helpers/Schema.test.js @@ -0,0 +1,243 @@ +require('module-alias/register') + +const Schema = require('@helpers/Schema') +const db = require('@config/database') + +afterAll(() => { + db.end() +}) + +afterEach(() => { + Schema.__reset() +}) + +describe('verifyScrapeSchema against the real dev schema', () => { + + test('reports ok once the migration has been applied', async () => { + const result = await Schema.verifyScrapeSchema() + + if (!result.ok) { + throw new Error( + `dev scrape schema not applied - run ./migrateDatabase.sh. ` + + `missing objects: ${result.missingObjects.join(', ') || 'none'}; ` + + `missing privileges: ${result.missingPrivileges.join(', ') || 'none'}` + ) + } + + expect(result.missingObjects).toEqual([]) + expect(result.missingPrivileges).toEqual([]) + expect(result.remedy).toBe('') + }) + + test('isScrapeSchemaReady reflects the last verification', async () => { + expect(Schema.isScrapeSchemaReady()).toBe(false) + + await Schema.verifyScrapeSchema() + expect(Schema.isScrapeSchemaReady()).toBe(true) + + Schema.__reset() + expect(Schema.isScrapeSchemaReady()).toBe(false) + }) + + test('__setReady is honoured, so request paths can be driven in tests', () => { + Schema.__setReady(true) + expect(Schema.isScrapeSchemaReady()).toBe(true) + Schema.__setReady(false) + expect(Schema.isScrapeSchemaReady()).toBe(false) + }) +}) + +// +// The FULLTEXT index is the one object whose loss is silent: MATCH..AGAINST +// against a BTREE index returns nothing without erroring. Guard it by type. +// +describe('FULLTEXT index guard', () => { + + test('attachment_text_index exists and is actually FULLTEXT', async () => { + await expect(Schema.__fulltextIndexExists('attachment', 'attachment_text_index')) + .resolves.toBe(true) + }) + + test('a BTREE index of the same shape would NOT satisfy the check', async () => { + // attachment_scrape_state is a real BTREE index, so this proves the + // check discriminates on type rather than just on name existing. + await expect(Schema.__indexExists('attachment', 'attachment_scrape_state')) + .resolves.toBe(true) + await expect(Schema.__fulltextIndexExists('attachment', 'attachment_scrape_state')) + .resolves.toBe(false) + }) + + test('verifyScrapeSchema includes it among the verified objects', async () => { + const result = await Schema.verifyScrapeSchema() + expect(result.ok).toBe(true) + expect(result.missingObjects).toEqual([]) + }) + + test('MATCH..AGAINST actually works, not just that an index exists', async () => { + // The functional end of the guard. An index of the right type that + // somehow could not be searched would still be a broken search. + const [rows] = await db.promise().query( + `SELECT COUNT(*) AS n FROM attachment + WHERE MATCH(text) AGAINST(? IN NATURAL LANGUAGE MODE)`, + ['the'] + ) + // Any result (including 0 matches) proves the query planner accepted + // MATCH..AGAINST against this column, which a BTREE index would reject. + expect(typeof rows[0].n).toBe('number') + }) +}) + +describe('object existence checks', () => { + + test('finds the real table, columns and index', async () => { + await expect(Schema.__tableExists('scrape_cache')).resolves.toBe(true) + await expect(Schema.__columnExists('attachment', 'scrape_state')).resolves.toBe(true) + await expect(Schema.__columnExists('attachment', 'scrape_attempts')).resolves.toBe(true) + await expect(Schema.__columnExists('attachment', 'enqueued_at')).resolves.toBe(true) + await expect(Schema.__indexExists('attachment', 'attachment_scrape_state')).resolves.toBe(true) + }) + + test('reports absence rather than throwing for things that do not exist', async () => { + await expect(Schema.__tableExists('no_such_table_xyz')).resolves.toBe(false) + await expect(Schema.__columnExists('attachment', 'no_such_column_xyz')).resolves.toBe(false) + await expect(Schema.__indexExists('attachment', 'no_such_index_xyz')).resolves.toBe(false) + }) +}) + +// +// The privilege check is what turns a future narrow-grants problem into one +// clear boot message instead of a swallowed runtime failure. +// +describe('parseGrants', () => { + + const DB = 'application' + const TABLES = ['attachment', 'scrape_cache'] + const parse = (lines) => Schema.parseGrants(lines, DB, TABLES) + + test('ALL PRIVILEGES ON *.* grants everything', () => { + const granted = parse(['GRANT ALL PRIVILEGES ON *.* TO `root`@`localhost` WITH GRANT OPTION']) + + expect(granted.has('SELECT')).toBe(true) + expect(granted.has('INSERT')).toBe(true) + expect(granted.has('UPDATE')).toBe(true) + expect(granted.has('DELETE')).toBe(true) + }) + + test('ALL PRIVILEGES scoped to the database grants everything', () => { + const granted = parse(['GRANT ALL PRIVILEGES ON `application`.* TO `app`@`localhost`']) + expect(granted.has('DELETE')).toBe(true) + }) + + test('reads an explicit privilege list', () => { + const granted = parse([ + 'GRANT USAGE ON *.* TO `app`@`localhost`', + 'GRANT SELECT, INSERT, UPDATE, DELETE ON `application`.* TO `app`@`localhost`', + ]) + + expect([...granted].sort()).toEqual(['DELETE', 'INSERT', 'SELECT', 'UPDATE']) + }) + + // The case that caused this whole change class. + test('spots a missing DELETE', () => { + const granted = parse(['GRANT SELECT, INSERT, UPDATE ON `application`.* TO `app`@`localhost`']) + + expect(granted.has('UPDATE')).toBe(true) + expect(granted.has('DELETE')).toBe(false) + }) + + test('accepts table-scoped grants for our tables', () => { + const granted = parse([ + 'GRANT SELECT, DELETE ON `application`.`attachment` TO `app`@`localhost`', + 'GRANT INSERT, UPDATE ON `application`.`scrape_cache` TO `app`@`localhost`', + ]) + + expect([...granted].sort()).toEqual(['DELETE', 'INSERT', 'SELECT', 'UPDATE']) + }) + + test('ignores grants on an unrelated database', () => { + const granted = parse(['GRANT ALL PRIVILEGES ON `somethingelse`.* TO `app`@`localhost`']) + expect(granted.size).toBe(0) + }) + + test('ignores USAGE, which conveys no privileges', () => { + const granted = parse(['GRANT USAGE ON *.* TO `app`@`localhost`']) + expect(granted.size).toBe(0) + }) + + test('handles grants written without backticks', () => { + const granted = parse(['GRANT SELECT, DELETE ON application.* TO app@localhost']) + expect(granted.has('DELETE')).toBe(true) + }) + + test('strips column-level qualifiers', () => { + const granted = parse(['GRANT SELECT (id, url), UPDATE (text) ON `application`.`attachment` TO `app`@`localhost`']) + + expect(granted.has('SELECT')).toBe(true) + expect(granted.has('UPDATE')).toBe(true) + }) + + test('ignores lines that are not grant statements', () => { + expect(parse(['', 'not a grant line', null]).size).toBe(0) + }) +}) + +// +// The regression that matters most. +// +// Schema.js used to issue CREATE TABLE / ALTER TABLE through the application's +// own connection pool, which failed on production's least-privilege user with +// "CREATE command denied to user 'app'@'localhost'". Nothing in this module may +// ever issue DDL again. +// +describe('issues no DDL', () => { + + test('verifyScrapeSchema runs only read-only statements', async () => { + const statements = [] + + // db.promise() hands back a NEW wrapper on every call, so spying on one + // instance would miss the one Schema.js obtains internally. Intercept + // the factory and instrument each wrapper it returns. + const realPromise = db.promise.bind(db) + const spy = jest.spyOn(db, 'promise').mockImplementation(() => { + const wrapper = realPromise() + const originalQuery = wrapper.query.bind(wrapper) + + wrapper.query = (sql, params) => { + statements.push(String(sql)) + return originalQuery(sql, params) + } + + return wrapper + }) + + await Schema.verifyScrapeSchema() + + spy.mockRestore() + + expect(statements.length).toBeGreaterThan(0) + + const forbidden = /\b(CREATE|ALTER|DROP|TRUNCATE|RENAME)\b/i + const offenders = statements.filter(statement => forbidden.test(statement)) + + expect(offenders).toEqual([]) + }) + + test('the module source contains no DDL outside comments', () => { + const fs = require('fs') + const path = require('path') + + const source = fs.readFileSync( + path.join(__dirname, '..', '..', 'helpers', 'Schema.js'), 'utf8' + ) + + const code = source + .split('\n') + .filter(line => !line.trim().startsWith('//')) + .join('\n') + + expect(code).not.toMatch(/\bCREATE\s+TABLE\b/i) + expect(code).not.toMatch(/\bALTER\s+TABLE\b/i) + expect(code).not.toMatch(/\bCREATE\s+INDEX\b/i) + expect(code).not.toMatch(/\bDROP\b/i) + }) +}) diff --git a/server/tests/schema/migrations.test.js b/server/tests/schema/migrations.test.js new file mode 100644 index 0000000..6d283fc --- /dev/null +++ b/server/tests/schema/migrations.test.js @@ -0,0 +1,220 @@ +require('module-alias/register') + +const { execFileSync } = require('child_process') +const fs = require('fs') +const path = require('path') + +const REPO = path.join(__dirname, '..', '..', '..') + +// +// These shell out to the Prisma CLI, which only exists inside nix-shell. Skip +// rather than fail when it is absent, so `npx jest` still works outside the +// shell -- but do NOT skip silently in a way that hides real drift when it IS +// available. +// +const hasPrisma = (() => { + try { + execFileSync('prisma', ['--version'], { stdio: 'ignore' }) + return true + } catch (error) { + return false + } +})() + +const describeWithPrisma = hasPrisma ? describe : describe.skip + +if (!hasPrisma) { + console.log('[schema tests] prisma not on PATH - skipping. Run inside nix-shell to exercise these.') +} + +const prisma = (args) => execFileSync('prisma', args, { + cwd: REPO, + encoding: 'utf8', + env: process.env, +}) + +describe('committed migration artifacts', () => { + + test('the baseline migration and lock file are present', () => { + expect(fs.existsSync(path.join(REPO, 'prisma/schema.prisma'))).toBe(true) + expect(fs.existsSync(path.join(REPO, 'prisma/migrations/0_init/migration.sql'))).toBe(true) + expect(fs.existsSync(path.join(REPO, 'prisma/migrations/migration_lock.toml'))).toBe(true) + }) + + // + // Prisma emits COLLATE utf8mb4_unicode_ci for MySQL, but every table in this + // database is utf8mb4_general_ci, and `prisma migrate diff` does NOT report + // the difference. So a regenerated migration would silently build a + // differently-collated database. This asserts the hand-correction is intact. + // + test('no migration declares a collation other than utf8mb4_general_ci', () => { + const dir = path.join(REPO, 'prisma/migrations') + const files = fs.readdirSync(dir, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => path.join(dir, entry.name, 'migration.sql')) + .filter(file => fs.existsSync(file)) + + expect(files.length).toBeGreaterThan(0) + + files.forEach(file => { + const ddl = fs.readFileSync(file, 'utf8') + .split('\n') + .filter(line => !line.trim().startsWith('--')) + .join('\n') + + const collations = [...ddl.matchAll(/COLLATE\s+([A-Za-z0-9_]+)/gi)].map(match => match[1]) + collations.forEach(collation => { + expect(collation).toBe('utf8mb4_general_ci') + }) + }) + }) + + test('the baseline creates the FULLTEXT index, not a plain one', () => { + const sql = fs.readFileSync(path.join(REPO, 'prisma/migrations/0_init/migration.sql'), 'utf8') + + expect(sql).toMatch(/FULLTEXT INDEX `attachment_text_index`/) + // A plain INDEX of that name would be the silent-failure mode. + expect(sql).not.toMatch(/^\s*INDEX `attachment_text_index`/m) + }) + + // + // @@fulltext is GA in Prisma 7 -- the `fullTextIndex` preview flag no longer + // exists in the CLI at all (verified: zero occurrences in the 7.8.0 bundle). + // It was silently ignored rather than rejected, which makes it exactly the + // kind of dead config that misleads the next reader. + // + test('the schema declares @@fulltext and no longer carries the dead preview flag', () => { + const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8') + + expect(schema).toMatch(/@@fulltext\(\[text\]/) + expect(schema).not.toMatch(/fullTextIndex/) + }) + + // + // No generator block means `prisma generate` cannot produce a client into + // this repo. That is the structural guarantee behind "migrations only" -- + // stronger than a convention, because it cannot be done by accident. + // + test('the schema declares no generator, so no client can be generated', () => { + const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8') + + expect(schema).not.toMatch(/generator\s+\w+\s*\{/) + expect(schema).toMatch(/datasource\s+db\s*\{/) + }) + + // + // The running server must never depend on Prisma. + // + test('no application code imports prisma', () => { + const { execFileSync } = require('child_process') + let hits = '' + try { + hits = execFileSync('grep', [ + '-rn', '--include=*.js', + '-E', "require\\(['\"]@?prisma", path.join(REPO, 'server'), + ], { encoding: 'utf8' }) + } catch (error) { + hits = '' // grep exits 1 when nothing matches + } + expect(hits.trim()).toBe('') + }) + + test('no @relation is introduced - this database has no foreign keys', () => { + // Adding relations would mean adding FK constraints to a live database, + // which is a schema change well outside adopting a migration tool. + const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8') + expect(schema).not.toMatch(/@relation/) + }) +}) + +describeWithPrisma('schema drift', () => { + + // + // The check that keeps schema.prisma honest: if the live database and the + // committed schema ever diverge, this fails. + // + test('schema.prisma matches the live database exactly', () => { + const output = prisma([ + 'migrate', 'diff', + '--from-config-datasource', + '--to-schema', 'prisma/schema.prisma', + '--script', + ]) + + expect(output).toMatch(/This is an empty migration/) + }) + + // + // The load-bearing guarantee: the committed migrations, applied to an empty + // database, reproduce the live schema exactly. + // + // Done by actually rebuilding into a scratch database and comparing + // information_schema, rather than via `migrate diff --from-migrations` + // (which requires a shadow database) -- and this is the stronger check + // anyway, because it also catches collation drift, which Prisma's own diff + // does NOT report. + // + test('the committed migrations rebuild the live schema exactly', () => { + const db = require('@config/database') + const SCRATCH = 'prisma_drift_check' + + const mysqlBin = (() => { + try { execFileSync('mariadb', ['--version'], { stdio: 'ignore' }); return 'mariadb' } + catch (error) { return 'mysql' } + })() + + const adminArgs = ['-u', 'root'] + const socket = path.join(REPO, '.mysql', 'mysql.sock') + if (fs.existsSync(socket)) { adminArgs.push(`--socket=${socket}`) } + + const admin = (sql, database) => execFileSync( + mysqlBin, + [...adminArgs, '-N', '-B', ...(database ? [database] : []), '-e', sql], + { encoding: 'utf8' } + ) + + try { + admin(`DROP DATABASE IF EXISTS \`${SCRATCH}\`; CREATE DATABASE \`${SCRATCH}\`;`) + + execFileSync('prisma', ['migrate', 'deploy'], { + cwd: REPO, + encoding: 'utf8', + env: { ...process.env, DATABASE_URL: `mysql://root@127.0.0.1:3306/${SCRATCH}` }, + }) + + const snapshot = (schema) => ({ + columns: admin( + `SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLLATION_NAME + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations' + ORDER BY TABLE_NAME, COLUMN_NAME` + ), + indexes: admin( + `SELECT TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX, COLUMN_NAME, NON_UNIQUE, INDEX_TYPE + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations' + ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX` + ), + collations: admin( + `SELECT TABLE_NAME, TABLE_COLLATION FROM information_schema.TABLES + WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations' + ORDER BY TABLE_NAME` + ), + }) + + const live = snapshot('application') + const rebuilt = snapshot(SCRATCH) + + expect(rebuilt.columns).toBe(live.columns) + expect(rebuilt.indexes).toBe(live.indexes) + expect(rebuilt.collations).toBe(live.collations) + } finally { + try { admin(`DROP DATABASE IF EXISTS \`${SCRATCH}\`;`) } catch (error) { /* best effort */ } + } + }) + + test('migrate status reports no pending migrations', () => { + const output = prisma(['migrate', 'status']) + expect(output).toMatch(/Database schema is up to date/) + }) +})