Testing Migrations

deploy.sh

@@ -63,21 +63,44 @@
 #
 # Then set DB_USER=app and DB_PASS=<password> in ~/.env. Note the asymmetry:
 # the admin scripts here (backupDatabase.sh, applyLastBackup.sh,
-# applyScrapeSchema.sh) still connect as root with $PROD_DB_PASS.
+# migrateDatabase.sh) still connect as root with $PROD_DB_PASS.
 #
 # DELETE is genuinely required -- the scraper drops shell attachment rows for
 # dead links and purges expired cache entries.
 #
 # --- schema changes ------------------------------------------------------
 #
+# Schema is managed by Prisma, for migrations and scaffolding ONLY -- the
+# running server never imports it and still talks to MariaDB through raw SQL.
+# The schema lives in prisma/schema.prisma; migrations in prisma/migrations/.
+#
 # Because the app holds no DDL, a deploy that introduces schema changes needs
 # one manual step, run once per environment as a database admin:
 #
-#   ./applyScrapeSchema.sh
+#   nix-shell shell.nix --run ./migrateDatabase.sh
 #
-# It is idempotent, so re-running is safe. The API verifies the schema and the
+# nix-shell is required: the prisma CLI comes from nixpkgs (prisma_7, 7.8.0),
-# grants at boot; if either is missing it logs the remedy and serves normally
+# not from node_modules. Alternatively add prisma_7 to the server's
-# with link previews disabled.
+# environment.systemPackages and call the script directly.
+#
+# It is idempotent, so re-running is safe. On a database that predates Prisma
+# the script detects the missing migration history and baselines it
+# automatically (marking 0_init applied rather than executing it) -- it does
+# NOT need to be done by hand.
+#
+# The API verifies the schema and the grants at boot; if either is missing it
+# logs the remedy and serves normally with link previews disabled.
+#
+# NEVER run `prisma migrate dev`, `prisma migrate reset` or `prisma db push`.
+# The first wants a shadow database; the latter two can drop data.
+#
+# DATABASE_URL is composed by the script from the admin credentials in ~/.env
+# and is never read by the application.
+#
+# Note: _prisma_migrations lives inside the `application` database, so the
+# least-privilege `app` user can write migration history (its grant is
+# database-wide). It holds no DDL, so it cannot reshape tables -- narrowing
+# this further would mean per-table grants, which is a separate project.
 #
 
 set -euo pipefail

server/helpers/Schema.js

@@ -10,7 +10,7 @@
 //
 // The restricted account is correct - the server ingests untrusted URLs and
 // should not be able to reshape its own tables. So DDL moved out of the
-// application entirely: an admin runs ./applyScrapeSchema.sh once, and this
+// application entirely: an admin runs ./migrateDatabase.sh once, and this
 // module only checks that the result is present and usable.
 //
 // Note on why the DDL is REMOVED rather than guarded by an existence check:
@@ -27,6 +27,17 @@ const REQUIRED_TABLE = 'scrape_cache'
 const REQUIRED_COLUMNS = ['scrape_state', 'scrape_attempts', 'enqueued_at']
 const REQUIRED_INDEX = 'attachment_scrape_state'
 
+//
+// The FULLTEXT index backing Attachment.textSearch's MATCH..AGAINST.
+//
+// Checked by TYPE, not just existence: Prisma's `@@fulltext` is a preview
+// feature and MariaDB is not among its documented databases, so a future
+// migration could plausibly emit a plain BTREE index of the same name. That
+// would make attachment search return nothing at all, with no error anywhere -
+// so it is worth asserting the type explicitly on every boot.
+//
+const REQUIRED_FULLTEXT = { table: 'attachment', index: 'attachment_text_index' }
+
 // The scraper writes and deletes, not just reads. DELETE is genuinely needed:
 // terminal failures drop the shell attachment row (ScrapeQueue), and
 // ScrapeCache.purgeExpired removes expired rows.
@@ -62,6 +73,19 @@ const columnExists = async (table, column) => {
 	return rows.length > 0
 }
 
+//
+// Does this index exist AND is it actually FULLTEXT?
+//
+const fulltextIndexExists = async (table, index) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.STATISTICS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?
+			AND INDEX_TYPE = 'FULLTEXT' LIMIT 1`,
+		[table, index]
+	)
+	return rows.length > 0
+}
+
 const indexExists = async (table, index) => {
 	const [rows] = await db.promise().query(
 		`SELECT 1 FROM information_schema.STATISTICS
@@ -74,6 +98,7 @@ const indexExists = async (table, index) => {
 Schema.__tableExists = tableExists
 Schema.__columnExists = columnExists
 Schema.__indexExists = indexExists
+Schema.__fulltextIndexExists = fulltextIndexExists
 
 //
 // Parse SHOW GRANTS output into the set of privileges that apply to this
@@ -182,12 +207,16 @@ Schema.verifyScrapeSchema = async () => {
 		missingObjects.push(`attachment index ${REQUIRED_INDEX}`)
 	}
 
+	if (!await fulltextIndexExists(REQUIRED_FULLTEXT.table, REQUIRED_FULLTEXT.index)) {
+		missingObjects.push(`FULLTEXT index ${REQUIRED_FULLTEXT.table}.${REQUIRED_FULLTEXT.index}`)
+	}
+
 	const privileges = await checkPrivileges()
 	const missingPrivileges = privileges.missing || []
 
 	const remedy = []
 	if (missingObjects.length) {
-		remedy.push('run ./applyScrapeSchema.sh as a database admin')
+		remedy.push('run ./migrateDatabase.sh as a database admin')
 	}
 	if (missingPrivileges.length) {
 		// Name the real database and user so this can be pasted straight into

server/index.js

@@ -350,7 +350,7 @@ app.use('/api/metric-tracking', metricTracking)
 //
 // The application deliberately holds no DDL privileges, so it does not create
 // its own schema - verifyScrapeSchema() only checks that an admin has already
-// run ./applyScrapeSchema.sh and that the required DML grants are present.
+// run ./migrateDatabase.sh and that the required DML grants are present.
 //
 // When anything is missing the scraper stays off rather than failing an INSERT
 // on every note save. The listener binds either way: the API must not depend on

server/migrations/0001_scrape_cache_and_state.sql

@@ -1,67 +0,0 @@
---
--- Scrape cache + per-attachment scrape lifecycle.
---
--- HOW THIS IS APPLIED
---
---   ./applyScrapeSchema.sh      (from the repo root, as an admin)
---
--- Run it once per environment. The application does NOT apply this itself:
--- production connects as a least-privilege `app` user with DML rights only, and
--- deliberately holds no CREATE/ALTER. server/helpers/Schema.js only *verifies*
--- that these objects exist and that the required privileges are held; if they
--- are missing it disables link previews and logs the remedy.
---
--- Safe to re-run: every statement is conditional.
---
--- Portability: dev and production are both MariaDB 11.4, so ADD COLUMN IF NOT
--- EXISTS and CREATE INDEX IF NOT EXISTS are available. If this project ever
--- moves to MySQL 8 these become invalid and need information_schema guards
--- instead.
---
-
--- One row per URL, shared across users and notes.
---
--- url_hash carries the UNIQUE index because a UNIQUE on varchar(1000) utf8mb4
--- is 4000 bytes and exceeds InnoDB's 3072-byte key limit. `url` is kept
--- human-readable for debugging only.
---
--- COLLATE is stated explicitly to match the existing tables (utf8mb4_general_ci)
--- rather than inheriting whatever the server default happens to be.
-CREATE TABLE IF NOT EXISTS `scrape_cache` (
-  `id`            int(10) unsigned NOT NULL AUTO_INCREMENT,
-  `url_hash`      char(64) NOT NULL,
-  `url`           varchar(1000) NOT NULL DEFAULT '',
-  `canonical_url` varchar(1000) DEFAULT NULL,
-  `status`        smallint(6) NOT NULL DEFAULT 0,
-  `title`         varchar(512) DEFAULT NULL,
-  `site_name`     varchar(255) DEFAULT NULL,
-  `description`   mediumtext DEFAULT NULL,
-  `keywords`      varchar(512) DEFAULT NULL,
-  `search_text`   mediumtext DEFAULT NULL,
-  `image_url`     varchar(1000) DEFAULT NULL,
-  `file_location` varchar(500) DEFAULT NULL,
-  `etag`          varchar(255) DEFAULT NULL,
-  `last_modified` varchar(128) DEFAULT NULL,
-  `content_type`  varchar(128) DEFAULT NULL,
-  `fetch_state`   varchar(20) NOT NULL DEFAULT 'pending',
-  `failure_count` int(11) NOT NULL DEFAULT 0,
-  `last_error`    varchar(255) DEFAULT NULL,
-  `fetched_at`    int(11) DEFAULT NULL,
-  `expires_at`    int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `scrape_cache_url_hash` (`url_hash`),
-  KEY `scrape_cache_expires_at` (`expires_at`),
-  KEY `scrape_cache_fetch_state` (`fetch_state`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-
--- Per-attachment lifecycle, so the in-memory scrape queue survives a restart.
---
--- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
---
--- DEFAULT 2 is load-bearing: it keeps every pre-existing attachment row
--- (~2200 in production) out of the boot recovery sweep.
-ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_state`    tinyint(4) NOT NULL DEFAULT 2;
-ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_attempts` tinyint(4) NOT NULL DEFAULT 0;
-ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `enqueued_at`     int(11) DEFAULT NULL;
-
-CREATE INDEX IF NOT EXISTS `attachment_scrape_state` ON `attachment` (`scrape_state`, `enqueued_at`);

server/tests/models/ScrapeQueue.test.js

@@ -39,7 +39,7 @@ beforeAll(async () => {
 	// The server normally does this at boot; tests have no boot phase.
 	const verification = await Schema.verifyScrapeSchema()
 	if (!verification.ok) {
-		throw new Error(`dev scrape schema not applied - run ./applyScrapeSchema.sh (${verification.remedy})`)
+		throw new Error(`dev scrape schema not applied - run ./migrateDatabase.sh (${verification.remedy})`)
 	}
 
 	try {

server/workers/ScrapeQueue.js

@@ -432,7 +432,7 @@ ScrapeQueue.recoverPending = async () => {
 	if (!settings.enabled) { return { requeued: 0, retired: 0 } }
 
 	// Every query below references scrape_state, which only exists once an
-	// admin has run ./applyScrapeSchema.sh.
+	// admin has run ./migrateDatabase.sh.
 	if (!Schema.isScrapeSchemaReady()) { return { requeued: 0, retired: 0 } }
 
 	// Anything left at FETCHING died with the previous process.

shell.nix

@@ -11,6 +11,17 @@ pkgs.mkShell {
     pm2
     mariadb
     openssl
+
+    # Schema migrations and DB scaffolding only -- the running server never
+    # imports Prisma. The nixpkgs wrapper is self-configuring: it exports
+    # PRISMA_SCHEMA_ENGINE_BINARY pointing at a nix-built prisma-engines and
+    # execs the bundled CLI, so there is no engine download and none of the
+    # usual NixOS prebuilt-binary trouble.
+    #
+    # Pinned to prisma_7 deliberately, NOT the `prisma` alias: Prisma 8 has no
+    # MariaDB support yet, so if nixpkgs moves that alias to v8 this project's
+    # database would stop being supported underneath us.
+    prisma_7
   ];
 
   shellHook = ''
@@ -79,8 +90,21 @@ pkgs.mkShell {
       done
       echo "MySQL ready."
 
+      #
+      # Collation is explicit because the server default here is
+      # utf8mb4_unicode_ci while every application table is
+      # utf8mb4_general_ci. Without this, a table created later without an
+      # explicit COLLATE would silently inherit the wrong one, changing string
+      # comparison and FULLTEXT ranking. Prisma does not model collation and
+      # `prisma migrate diff` does not report the difference.
+      #
+      # Existing databases need a one-time, non-destructive fix:
+      #   ALTER DATABASE application
+      #     CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+      #
       mysql -u root --socket="$MYSQL_SOCKET" \
-        -e "CREATE DATABASE IF NOT EXISTS application;" 2>/dev/null
+        -e "CREATE DATABASE IF NOT EXISTS application \
+              CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" 2>/dev/null
     fi
 
     #
@@ -98,6 +122,24 @@ pkgs.mkShell {
       export DB_USER=root
       export DB_PASS=
 
+      #
+      # Prisma CLI connection string, for migrations and scaffolding.
+      #
+      # The warning above about shadowing ~/.env does NOT apply here:
+      # DATABASE_URL is read only by the Prisma CLI, never by
+      # server/config/database.js, so it cannot affect the running server.
+      #
+      # Deliberately NOT derived from DB_USER/DB_PASS. Prisma needs DDL
+      # rights, and in production those variables hold the least-privilege
+      # `app` account. Admin credentials are composed by migrateDatabase.sh
+      # instead, which is why this export lives only in the dev branch.
+      #
+      export DATABASE_URL="mysql://root@127.0.0.1:3306/application"
+
+      # Stop the Prisma CLI phoning checkpoint.prisma.io for update checks.
+      # Keeps the schema tests network-free and the output uncluttered.
+      export CHECKPOINT_DISABLE=1
+
       # Stable dev JWT key, persisted across shell sessions
       if [ -z "''${JSON_KEY:-}" ]; then
         if [ -s "$PWD/.mysql/jwt_key" ]; then