diff --git a/deploy.sh b/deploy.sh index dc8d2ab..4e06b6d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -63,21 +63,44 @@ # # Then set DB_USER=app and DB_PASS= in ~/.env. Note the asymmetry: # the admin scripts here (backupDatabase.sh, applyLastBackup.sh, -# applyScrapeSchema.sh) still connect as root with $PROD_DB_PASS. +# migrateDatabase.sh) still connect as root with $PROD_DB_PASS. # # DELETE is genuinely required -- the scraper drops shell attachment rows for # dead links and purges expired cache entries. # # --- schema changes ------------------------------------------------------ # +# Schema is managed by Prisma, for migrations and scaffolding ONLY -- the +# running server never imports it and still talks to MariaDB through raw SQL. +# The schema lives in prisma/schema.prisma; migrations in prisma/migrations/. +# # Because the app holds no DDL, a deploy that introduces schema changes needs # one manual step, run once per environment as a database admin: # -# ./applyScrapeSchema.sh +# nix-shell shell.nix --run ./migrateDatabase.sh # -# It is idempotent, so re-running is safe. The API verifies the schema and the -# grants at boot; if either is missing it logs the remedy and serves normally -# with link previews disabled. +# nix-shell is required: the prisma CLI comes from nixpkgs (prisma_7, 7.8.0), +# not from node_modules. Alternatively add prisma_7 to the server's +# environment.systemPackages and call the script directly. +# +# It is idempotent, so re-running is safe. On a database that predates Prisma +# the script detects the missing migration history and baselines it +# automatically (marking 0_init applied rather than executing it) -- it does +# NOT need to be done by hand. +# +# The API verifies the schema and the grants at boot; if either is missing it +# logs the remedy and serves normally with link previews disabled. +# +# NEVER run `prisma migrate dev`, `prisma migrate reset` or `prisma db push`. +# The first wants a shadow database; the latter two can drop data. +# +# DATABASE_URL is composed by the script from the admin credentials in ~/.env +# and is never read by the application. +# +# Note: _prisma_migrations lives inside the `application` database, so the +# least-privilege `app` user can write migration history (its grant is +# database-wide). It holds no DDL, so it cannot reshape tables -- narrowing +# this further would mean per-table grants, which is a separate project. # set -euo pipefail diff --git a/server/helpers/Schema.js b/server/helpers/Schema.js index 47cf649..6f7433d 100644 --- a/server/helpers/Schema.js +++ b/server/helpers/Schema.js @@ -10,7 +10,7 @@ // // The restricted account is correct - the server ingests untrusted URLs and // should not be able to reshape its own tables. So DDL moved out of the -// application entirely: an admin runs ./applyScrapeSchema.sh once, and this +// application entirely: an admin runs ./migrateDatabase.sh once, and this // module only checks that the result is present and usable. // // Note on why the DDL is REMOVED rather than guarded by an existence check: @@ -27,6 +27,17 @@ const REQUIRED_TABLE = 'scrape_cache' const REQUIRED_COLUMNS = ['scrape_state', 'scrape_attempts', 'enqueued_at'] const REQUIRED_INDEX = 'attachment_scrape_state' +// +// The FULLTEXT index backing Attachment.textSearch's MATCH..AGAINST. +// +// Checked by TYPE, not just existence: Prisma's `@@fulltext` is a preview +// feature and MariaDB is not among its documented databases, so a future +// migration could plausibly emit a plain BTREE index of the same name. That +// would make attachment search return nothing at all, with no error anywhere - +// so it is worth asserting the type explicitly on every boot. +// +const REQUIRED_FULLTEXT = { table: 'attachment', index: 'attachment_text_index' } + // The scraper writes and deletes, not just reads. DELETE is genuinely needed: // terminal failures drop the shell attachment row (ScrapeQueue), and // ScrapeCache.purgeExpired removes expired rows. @@ -62,6 +73,19 @@ const columnExists = async (table, column) => { return rows.length > 0 } +// +// Does this index exist AND is it actually FULLTEXT? +// +const fulltextIndexExists = async (table, index) => { + const [rows] = await db.promise().query( + `SELECT 1 FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? + AND INDEX_TYPE = 'FULLTEXT' LIMIT 1`, + [table, index] + ) + return rows.length > 0 +} + const indexExists = async (table, index) => { const [rows] = await db.promise().query( `SELECT 1 FROM information_schema.STATISTICS @@ -74,6 +98,7 @@ const indexExists = async (table, index) => { Schema.__tableExists = tableExists Schema.__columnExists = columnExists Schema.__indexExists = indexExists +Schema.__fulltextIndexExists = fulltextIndexExists // // Parse SHOW GRANTS output into the set of privileges that apply to this @@ -182,12 +207,16 @@ Schema.verifyScrapeSchema = async () => { missingObjects.push(`attachment index ${REQUIRED_INDEX}`) } + if (!await fulltextIndexExists(REQUIRED_FULLTEXT.table, REQUIRED_FULLTEXT.index)) { + missingObjects.push(`FULLTEXT index ${REQUIRED_FULLTEXT.table}.${REQUIRED_FULLTEXT.index}`) + } + const privileges = await checkPrivileges() const missingPrivileges = privileges.missing || [] const remedy = [] if (missingObjects.length) { - remedy.push('run ./applyScrapeSchema.sh as a database admin') + remedy.push('run ./migrateDatabase.sh as a database admin') } if (missingPrivileges.length) { // Name the real database and user so this can be pasted straight into diff --git a/server/index.js b/server/index.js index 7ae398c..3bf4536 100644 --- a/server/index.js +++ b/server/index.js @@ -350,7 +350,7 @@ app.use('/api/metric-tracking', metricTracking) // // The application deliberately holds no DDL privileges, so it does not create // its own schema - verifyScrapeSchema() only checks that an admin has already -// run ./applyScrapeSchema.sh and that the required DML grants are present. +// run ./migrateDatabase.sh and that the required DML grants are present. // // When anything is missing the scraper stays off rather than failing an INSERT // on every note save. The listener binds either way: the API must not depend on diff --git a/server/migrations/0001_scrape_cache_and_state.sql b/server/migrations/0001_scrape_cache_and_state.sql deleted file mode 100644 index 3153c7d..0000000 --- a/server/migrations/0001_scrape_cache_and_state.sql +++ /dev/null @@ -1,67 +0,0 @@ --- --- Scrape cache + per-attachment scrape lifecycle. --- --- HOW THIS IS APPLIED --- --- ./applyScrapeSchema.sh (from the repo root, as an admin) --- --- Run it once per environment. The application does NOT apply this itself: --- production connects as a least-privilege `app` user with DML rights only, and --- deliberately holds no CREATE/ALTER. server/helpers/Schema.js only *verifies* --- that these objects exist and that the required privileges are held; if they --- are missing it disables link previews and logs the remedy. --- --- Safe to re-run: every statement is conditional. --- --- Portability: dev and production are both MariaDB 11.4, so ADD COLUMN IF NOT --- EXISTS and CREATE INDEX IF NOT EXISTS are available. If this project ever --- moves to MySQL 8 these become invalid and need information_schema guards --- instead. --- - --- One row per URL, shared across users and notes. --- --- url_hash carries the UNIQUE index because a UNIQUE on varchar(1000) utf8mb4 --- is 4000 bytes and exceeds InnoDB's 3072-byte key limit. `url` is kept --- human-readable for debugging only. --- --- COLLATE is stated explicitly to match the existing tables (utf8mb4_general_ci) --- rather than inheriting whatever the server default happens to be. -CREATE TABLE IF NOT EXISTS `scrape_cache` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `url_hash` char(64) NOT NULL, - `url` varchar(1000) NOT NULL DEFAULT '', - `canonical_url` varchar(1000) DEFAULT NULL, - `status` smallint(6) NOT NULL DEFAULT 0, - `title` varchar(512) DEFAULT NULL, - `site_name` varchar(255) DEFAULT NULL, - `description` mediumtext DEFAULT NULL, - `keywords` varchar(512) DEFAULT NULL, - `search_text` mediumtext DEFAULT NULL, - `image_url` varchar(1000) DEFAULT NULL, - `file_location` varchar(500) DEFAULT NULL, - `etag` varchar(255) DEFAULT NULL, - `last_modified` varchar(128) DEFAULT NULL, - `content_type` varchar(128) DEFAULT NULL, - `fetch_state` varchar(20) NOT NULL DEFAULT 'pending', - `failure_count` int(11) NOT NULL DEFAULT 0, - `last_error` varchar(255) DEFAULT NULL, - `fetched_at` int(11) DEFAULT NULL, - `expires_at` int(11) DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `scrape_cache_url_hash` (`url_hash`), - KEY `scrape_cache_expires_at` (`expires_at`), - KEY `scrape_cache_fetch_state` (`fetch_state`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; - --- Per-attachment lifecycle, so the in-memory scrape queue survives a restart. --- --- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed. --- --- DEFAULT 2 is load-bearing: it keeps every pre-existing attachment row --- (~2200 in production) out of the boot recovery sweep. -ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_state` tinyint(4) NOT NULL DEFAULT 2; -ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_attempts` tinyint(4) NOT NULL DEFAULT 0; -ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `enqueued_at` int(11) DEFAULT NULL; - -CREATE INDEX IF NOT EXISTS `attachment_scrape_state` ON `attachment` (`scrape_state`, `enqueued_at`); diff --git a/server/tests/models/ScrapeQueue.test.js b/server/tests/models/ScrapeQueue.test.js index 6f26896..938f492 100644 --- a/server/tests/models/ScrapeQueue.test.js +++ b/server/tests/models/ScrapeQueue.test.js @@ -39,7 +39,7 @@ beforeAll(async () => { // The server normally does this at boot; tests have no boot phase. const verification = await Schema.verifyScrapeSchema() if (!verification.ok) { - throw new Error(`dev scrape schema not applied - run ./applyScrapeSchema.sh (${verification.remedy})`) + throw new Error(`dev scrape schema not applied - run ./migrateDatabase.sh (${verification.remedy})`) } try { diff --git a/server/workers/ScrapeQueue.js b/server/workers/ScrapeQueue.js index ba8d8f6..5cc35b8 100644 --- a/server/workers/ScrapeQueue.js +++ b/server/workers/ScrapeQueue.js @@ -432,7 +432,7 @@ ScrapeQueue.recoverPending = async () => { if (!settings.enabled) { return { requeued: 0, retired: 0 } } // Every query below references scrape_state, which only exists once an - // admin has run ./applyScrapeSchema.sh. + // admin has run ./migrateDatabase.sh. if (!Schema.isScrapeSchemaReady()) { return { requeued: 0, retired: 0 } } // Anything left at FETCHING died with the previous process. diff --git a/shell.nix b/shell.nix index bd2567e..ff4148b 100644 --- a/shell.nix +++ b/shell.nix @@ -11,6 +11,17 @@ pkgs.mkShell { pm2 mariadb openssl + + # Schema migrations and DB scaffolding only -- the running server never + # imports Prisma. The nixpkgs wrapper is self-configuring: it exports + # PRISMA_SCHEMA_ENGINE_BINARY pointing at a nix-built prisma-engines and + # execs the bundled CLI, so there is no engine download and none of the + # usual NixOS prebuilt-binary trouble. + # + # Pinned to prisma_7 deliberately, NOT the `prisma` alias: Prisma 8 has no + # MariaDB support yet, so if nixpkgs moves that alias to v8 this project's + # database would stop being supported underneath us. + prisma_7 ]; shellHook = '' @@ -79,8 +90,21 @@ pkgs.mkShell { done echo "MySQL ready." + # + # Collation is explicit because the server default here is + # utf8mb4_unicode_ci while every application table is + # utf8mb4_general_ci. Without this, a table created later without an + # explicit COLLATE would silently inherit the wrong one, changing string + # comparison and FULLTEXT ranking. Prisma does not model collation and + # `prisma migrate diff` does not report the difference. + # + # Existing databases need a one-time, non-destructive fix: + # ALTER DATABASE application + # CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + # mysql -u root --socket="$MYSQL_SOCKET" \ - -e "CREATE DATABASE IF NOT EXISTS application;" 2>/dev/null + -e "CREATE DATABASE IF NOT EXISTS application \ + CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" 2>/dev/null fi # @@ -98,6 +122,24 @@ pkgs.mkShell { export DB_USER=root export DB_PASS= + # + # Prisma CLI connection string, for migrations and scaffolding. + # + # The warning above about shadowing ~/.env does NOT apply here: + # DATABASE_URL is read only by the Prisma CLI, never by + # server/config/database.js, so it cannot affect the running server. + # + # Deliberately NOT derived from DB_USER/DB_PASS. Prisma needs DDL + # rights, and in production those variables hold the least-privilege + # `app` account. Admin credentials are composed by migrateDatabase.sh + # instead, which is why this export lives only in the dev branch. + # + export DATABASE_URL="mysql://root@127.0.0.1:3306/application" + + # Stop the Prisma CLI phoning checkpoint.prisma.io for update checks. + # Keeps the schema tests network-free and the output uncluttered. + export CHECKPOINT_DISABLE=1 + # Stable dev JWT key, persisted across shell sessions if [ -z "''${JSON_KEY:-}" ]; then if [ -s "$PWD/.mysql/jwt_key" ]; then