From cfc1aaf2d57bfd97f45347325f8c2a123b715199 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 20 Aug 2026 11:59:13 -0700 Subject: [PATCH 1/4] ADFA-5212: Add a corrected docdb script for the Dynamic Bookshelf The three prototypes attached to the ticket do not run. Both table scripts write `CREATE TABLE IF NOT EXISTS`, where SQLite wants the clause before the name, and the template script has no IF NOT EXISTS at all, so `CREATE TABLE Templates` fails against any real database. Worse than failing, they half-apply: the sqlite3 CLI reports each error, carries on, and reaches COMMIT. Running them against a copy of the 14-Aug database left 22 Bookshelf rows -- the 7 good books on top of the 15 broken ones -- and updated none of the existing category descriptions, because ids 1-5 collided on the primary key. This replaces all three with one script, since the sections depend on each other and share a safety harness: - `.bail on`, so an error aborts instead of persisting partial work. - Idempotent. Categories are inserted if missing and their descriptions refreshed, never re-keyed, because Bookshelf.bookCategoryID points at those ids. Books are rebuilt. The template is updated in place if present. - No hard-coded Content.id. The prototype's own comment warned those would be wrong elsewhere; they are AUTOINCREMENT values assigned at import. Books resolve by Content.path, which is stable across rebuilds and fails safe: a path that is missing inserts nothing rather than attaching a book to whatever row now holds that id. - Verification that names what broke. SQLite prohibits subqueries in CHECK, so violations are collected into a temp table, printed, and then gated on a CHECK that fails the transaction. The template blob is the revision verified on the device on 19-Aug: 1,261 bytes, debug output removed, category names matching the seed data. Tested against a copy of the 14-Aug database: converges from 5 categories / 15 unusable rows / no template to 6 / 7 / installed, is unchanged by two further runs, and rolls back with a named diagnostic when a book path is missing from Content or the template blob is truncated. Two lessons added to docs/documentation-database.md: never hard-code a Content.id, and how to write a row-counting invariant given that CHECK cannot hold a subquery (including the HAVING that keeps an aggregate check from firing on a clean run). --- docs/docdb/ADFA-5212-dynamic-bookshelf.sql | 258 +++++++++++++++++++++ docs/documentation-database.md | 4 +- 2 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 docs/docdb/ADFA-5212-dynamic-bookshelf.sql diff --git a/docs/docdb/ADFA-5212-dynamic-bookshelf.sql b/docs/docdb/ADFA-5212-dynamic-bookshelf.sql new file mode 100644 index 0000000000..e8f6f2c901 --- /dev/null +++ b/docs/docdb/ADFA-5212-dynamic-bookshelf.sql @@ -0,0 +1,258 @@ +-- ADFA-5212: Dynamic Bookshelf tables, data, and template. +-- +-- Creates BookCategories and Bookshelf if they are missing, fills them with +-- the books currently shipped by the bookshelf plugin, and installs the +-- 'bookshelf' Pebble template that WebServer renders at /pr/bs. +-- +-- Supersedes the three prototype scripts attached to ADFA-5212. Those had +-- `CREATE TABLE IF NOT EXISTS` (SQLite wants that clause before the +-- name, so each was a parse error), no IF NOT EXISTS at all on Templates, and +-- hard-coded Content.id values. One file rather than three because the +-- sections depend on each other and share the safety harness below. +-- +-- Apply against the real documentation.db: +-- sqlite3 documentation.db < ADFA-5212-dynamic-bookshelf.sql +-- +-- Two properties the prototypes did not have: +-- +-- * Idempotent. Running it twice leaves the same rows: categories are +-- updated in place (ids preserved, because Bookshelf references them), +-- the book list is rebuilt, and the template is updated if already there. +-- +-- * No hard-coded Content.id. Those are AUTOINCREMENT values assigned at +-- import time and differ in every rebuild of the database, which is what +-- the prototype's own comment warned about. Books resolve by Content.path +-- instead, which is stable, and a book whose path is missing from Content +-- is simply not inserted rather than pointing at whichever row happens to +-- hold that id. +-- +-- `.bail on` matters more than it looks: without it the sqlite3 CLI reports an +-- error, carries on, and reaches COMMIT anyway, persisting whatever succeeded +-- (see docs/documentation-database.md). Running the prototypes against a copy +-- of the 14-Aug database did exactly that, leaving 22 Bookshelf rows: the 7 +-- good books on top of the 15 broken ones. + +.bail on + +-- Enforced so an inserted bookCategoryID that does not resolve is an error +-- here rather than an empty bookshelf later. Must be set outside the +-- transaction; SQLite ignores it inside one. +PRAGMA foreign_keys = ON; + +BEGIN TRANSACTION; + +-- --------------------------------------------------------------------- +-- Tables +-- --------------------------------------------------------------------- +-- Definitions match the tables already present in shipped databases, so an +-- existing database keeps its schema and only the data below changes. + +CREATE TABLE IF NOT EXISTS BookCategories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT, + description TEXT DEFAULT '', + UNIQUE(category) +); + +CREATE TABLE IF NOT EXISTS Bookshelf ( + contentID INTEGER NOT NULL, + title TEXT DEFAULT '', + description TEXT DEFAULT '', + bookCategoryID INTEGER, + FOREIGN KEY (bookCategoryID) REFERENCES BookCategories(id), + UNIQUE(title, bookCategoryID) +); + +-- --------------------------------------------------------------------- +-- Seed data +-- --------------------------------------------------------------------- +-- In temp tables so the sections below can join on them and the verification +-- at the end can compare against what was asked for. +-- +-- Descriptions are inserted into HTML by the template, so the entities are +-- deliberate. The prototype had a bare '&' in the Kotlin entry; it is '&' +-- here, matching the others. + +CREATE TEMP TABLE CategorySeed ( + category TEXT NOT NULL, + description TEXT NOT NULL +); + +INSERT INTO CategorySeed (category, description) VALUES + ('General', 'Books about computing'), + ('Java', 'Books about the Java programming language'), + ('Kotlin', 'Books about the Kotlin programming language'), + ('Pebble', 'Books about the Pebble template mini-language'), + ('Android', 'Books about Android mobile programming'), + ('C and C++', 'Books about the C and C++ programming languages and the C Preprocessor'); + +CREATE TEMP TABLE BookSeed ( + path TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + category TEXT NOT NULL +); + +INSERT INTO BookSeed (path, title, description, category) VALUES + ('bookshelf/org.appdevforall.bookshelfplugin/AndroidNotesForProfessionals.pdf', + 'Android Notes for Professionals', + 'Compiled from Stack Overflow. 266 chapters, 1297 pages covers Activities, Fragments, RecyclerViews, JSON parsing, Background Tasks, and more', + 'Android'), + ('bookshelf/org.appdevforall.bookshelfplugin/BeejGuideToCProgramming.pdf', + 'Beej Guide To C Programming', + 'By Brian “Beej Jorgensen” Hall. 41 chapters, 342 pages covers C, the preprocessor, and the standard library', + 'C and C++'), + ('bookshelf/org.appdevforall.bookshelfplugin/JavaJavaJava.pdf', + 'Java, Java, Java: Object-Oriented Problem Solving', + 'By Ralph Morelli and Ralph Wade. 16 chapters and 8 appendices, 840 pages covers basic Java including exceptions, also graphics, threads, and socket programming', + 'Java'), + ('bookshelf/org.appdevforall.bookshelfplugin/JavaNotesForProfessionals.pdf', + 'Java Notes for Professionals', + 'Compiled from Stack Overflow. 181 chapters and 4 appendices, 951 pages covering Syntax, Semantics, Compilation, Documentation, Libraries, Generics, and more', + 'Java'), + ('bookshelf/org.appdevforall.bookshelfplugin/KotlinNotesForProfessionals.pdf', + 'Kotlin Notes for Professionals', + 'Compiled from Stack Overflow. 94 pages, covering Basics & Control Flow, Null Safety, Object-Oriented & Functional Mix, Advanced Features, Java Interoperability, Android Specifics, and more', + 'Kotlin'), + ('bookshelf/org.appdevforall.bookshelfplugin/ModernCplusplusTutorialOuChangkun.pdf', + 'Modern C++ Tutorial', + 'By Ou Changkun. 10 chapters and 2 appendices, 111 pages, “C++ programmers who are still using traditional C++ (this book refers to C++98 and its previous standards as traditional C++) may even amazed by the fact that they are not using the same language while reading modern C++ code.”', + 'C and C++'), + ('bookshelf/org.appdevforall.bookshelfplugin/PebbleTemplateGuide.pdf', + 'Pebble Template Guide', + '89 pages, a PDF version of the website', + 'Pebble'); + +-- --------------------------------------------------------------------- +-- BookCategories +-- --------------------------------------------------------------------- +-- Insert the missing ones, then refresh every description. Ids are never +-- reassigned: Bookshelf.bookCategoryID points at them, and INSERT OR REPLACE +-- would delete and re-add the row with a new id. + +INSERT INTO BookCategories (category, description) +SELECT S.category, S.description + FROM CategorySeed S + WHERE NOT EXISTS (SELECT 1 FROM BookCategories BC WHERE BC.category = S.category); + +UPDATE BookCategories + SET description = (SELECT S.description FROM CategorySeed S WHERE S.category = BookCategories.category) + WHERE category IN (SELECT category FROM CategorySeed); + +-- --------------------------------------------------------------------- +-- Bookshelf +-- --------------------------------------------------------------------- +-- Rebuilt rather than merged. The 14-Aug database's rows are unusable (NULL +-- bookCategoryID, datetime-placeholder titles; see ADFA-5204), and adding to +-- them leaves real books beside placeholder ones. If a database ever carries +-- bookshelf rows from another source, narrow this DELETE to the seeded paths. + +DELETE FROM Bookshelf; + +INSERT INTO Bookshelf (contentID, title, description, bookCategoryID) +SELECT C.id, S.title, S.description, BC.id + FROM BookSeed S + JOIN Content C ON C.path = S.path + JOIN BookCategories BC ON BC.category = S.category; + +-- --------------------------------------------------------------------- +-- Templates: the 'bookshelf' Pebble template +-- --------------------------------------------------------------------- +-- WebServer looks this row up by name, so its id does not matter -- but the id +-- is left alone on an update, because Content.templateId refers to template +-- ids numerically and reassigning them is a hazard worth avoiding entirely. +-- +-- The blob is the template verified on device on 19-Aug: 1261 bytes, with the +-- debug output removed (it was most of the rendered page) and its five +-- category names matching the seed data above. Those names are hardcoded in +-- the template on purpose, to control display order without a sort column in +-- BookCategories, so a seventh category needs a template edit too. + +INSERT INTO Templates (name, content) +SELECT 'bookshelf', X'3c21444f43545950452068746d6c3e0a3c68746d6c206c616e673d22656e2d7573223e0a3c686561643e0a3c7374796c6520747970653d22746578742f637373223e0a2e66696c656c696e6b207b206261636b67726f756e642d636f6c6f723a2079656c6c6f773b207d0a2e7765626c696e6b207b20206261636b67726f756e642d636f6c6f723a206379616e3b7d0a3c2f7374796c653e0a7b25206d6163726f20657870616e64426f6f6b7328726573756c742c2063617465676f72792920257d0a20207b2520666f72206974656d20696e20726573756c7420257d0a202020207b25206966206974656d2e63617465676f7279203d3d2063617465676f727920257d0a3c68313e43617465676f72793a207b7b2063617465676f7279207d7d3c2f68313e0a7b25206966206974656d2e6465736372697074696f6e20213d20272720257d0a3c68323e4465736372697074696f6e3a207b7b206974656d2e6465736372697074696f6e207d7d3c2f68323e0a7b2520656e64696620257d0a2020202020207b2520666f7220626f6f6b20696e206974656d2e626f6f6b7320257d0a3c703e0a20202020202020207b2520696620626f6f6b2e706466203d3d203120257d0a20203c6120687265663d222f702f7765622f7669657765722e68746d6c3f66696c653d2f7b7b20626f6f6b2e6c696e6b207d7d22207461726765743d225f626c616e6b2220636c6173733d2266696c656c696e6b223e7b7b20626f6f6b2e7469746c65207d7d3c2f613e0a202020202020202020207b2520696620626f6f6b2e6465736372697074696f6e20213d20272720257d0a202020202020202020202020287b7b20626f6f6b2e6465736372697074696f6e207d7d203c693e5044463c2f693e290a202020202020202020207b2520656e64696620257d0a20202020202020207b2520656c736520257d0a20203c6120687265663d222f7b7b20626f6f6b2e6c696e6b207d7d22207461726765743d225f626c616e6b2220636c6173733d227765626c696e6b223e7b7b20626f6f6b2e7469746c65207d7d3c2f613e0a202020202020202020207b2520696620626f6f6b2e6465736372697074696f6e20213d20272720257d0a202020202020202020202020287b7b20626f6f6b2e6465736372697074696f6e207d7d290a202020202020202020207b2520656e64696620257d0a20202020202020207b2520656e64696620257d0a3c2f703e0a2020202020207b2520656e64666f7220257d0a202020207b2520656e64696620257d0a20207b2520656e64666f7220257d0a7b2520656e646d6163726f20257d0a0a3c2f686561643e3c626f64793e3c703e54686520666f6c6c6f77696e6720626f6f6b7320616e64207265666572656e6365206d6174657269616c732061726520696e636c75646564207769746820436f6465206f6e2074686520476f2e3c2f703e0a7b7b20657870616e64426f6f6b7328726573756c742c2027416e64726f69642729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274a6176612729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274b6f746c696e2729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274320616e6420432b2b2729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c2027506562626c652729207d7d0a3c2f626f64793e3c2f68746d6c3e0a' + WHERE NOT EXISTS (SELECT 1 FROM Templates WHERE name = 'bookshelf'); + +UPDATE Templates + SET content = X'3c21444f43545950452068746d6c3e0a3c68746d6c206c616e673d22656e2d7573223e0a3c686561643e0a3c7374796c6520747970653d22746578742f637373223e0a2e66696c656c696e6b207b206261636b67726f756e642d636f6c6f723a2079656c6c6f773b207d0a2e7765626c696e6b207b20206261636b67726f756e642d636f6c6f723a206379616e3b7d0a3c2f7374796c653e0a7b25206d6163726f20657870616e64426f6f6b7328726573756c742c2063617465676f72792920257d0a20207b2520666f72206974656d20696e20726573756c7420257d0a202020207b25206966206974656d2e63617465676f7279203d3d2063617465676f727920257d0a3c68313e43617465676f72793a207b7b2063617465676f7279207d7d3c2f68313e0a7b25206966206974656d2e6465736372697074696f6e20213d20272720257d0a3c68323e4465736372697074696f6e3a207b7b206974656d2e6465736372697074696f6e207d7d3c2f68323e0a7b2520656e64696620257d0a2020202020207b2520666f7220626f6f6b20696e206974656d2e626f6f6b7320257d0a3c703e0a20202020202020207b2520696620626f6f6b2e706466203d3d203120257d0a20203c6120687265663d222f702f7765622f7669657765722e68746d6c3f66696c653d2f7b7b20626f6f6b2e6c696e6b207d7d22207461726765743d225f626c616e6b2220636c6173733d2266696c656c696e6b223e7b7b20626f6f6b2e7469746c65207d7d3c2f613e0a202020202020202020207b2520696620626f6f6b2e6465736372697074696f6e20213d20272720257d0a202020202020202020202020287b7b20626f6f6b2e6465736372697074696f6e207d7d203c693e5044463c2f693e290a202020202020202020207b2520656e64696620257d0a20202020202020207b2520656c736520257d0a20203c6120687265663d222f7b7b20626f6f6b2e6c696e6b207d7d22207461726765743d225f626c616e6b2220636c6173733d227765626c696e6b223e7b7b20626f6f6b2e7469746c65207d7d3c2f613e0a202020202020202020207b2520696620626f6f6b2e6465736372697074696f6e20213d20272720257d0a202020202020202020202020287b7b20626f6f6b2e6465736372697074696f6e207d7d290a202020202020202020207b2520656e64696620257d0a20202020202020207b2520656e64696620257d0a3c2f703e0a2020202020207b2520656e64666f7220257d0a202020207b2520656e64696620257d0a20207b2520656e64666f7220257d0a7b2520656e646d6163726f20257d0a0a3c2f686561643e3c626f64793e3c703e54686520666f6c6c6f77696e6720626f6f6b7320616e64207265666572656e6365206d6174657269616c732061726520696e636c75646564207769746820436f6465206f6e2074686520476f2e3c2f703e0a7b7b20657870616e64426f6f6b7328726573756c742c2027416e64726f69642729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274a6176612729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274b6f746c696e2729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274320616e6420432b2b2729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c2027506562626c652729207d7d0a3c2f626f64793e3c2f68746d6c3e0a' + WHERE name = 'bookshelf'; + +-- --------------------------------------------------------------------- +-- Verification +-- --------------------------------------------------------------------- +-- Each statement records a row only when its invariant is violated, so a clean +-- run collects nothing. The SELECT then prints whatever was collected -- naming +-- the problem, which a bare CHECK failure would not -- and the gate turns a +-- non-empty list into an error `.bail on` acts on, rolling the transaction back. +-- +-- Written this way because SQLite prohibits subqueries inside CHECK, so the +-- conditions have to live in INSERT ... WHERE. + +CREATE TEMP TABLE Problems ( + problem TEXT NOT NULL +); + +-- Every seeded book resolved to a Content row. This is the check that catches a +-- database whose Content.path values differ from the seed list. +INSERT INTO Problems (problem) +SELECT 'these seeded book paths are missing from Content: ' || GROUP_CONCAT(path, '; ') + FROM BookSeed S + WHERE NOT EXISTS (SELECT 1 FROM Content C WHERE C.path = S.path) +-- Without this the aggregate still returns one row on a clean run, GROUP_CONCAT +-- is NULL, and the insert fails where nothing is wrong. +HAVING COUNT(*) > 0; + +-- The bookshelf holds exactly the seeded books. +INSERT INTO Problems (problem) +SELECT 'Bookshelf has ' || (SELECT COUNT(*) FROM Bookshelf) || ' rows, expected ' || (SELECT COUNT(*) FROM BookSeed) + WHERE (SELECT COUNT(*) FROM Bookshelf) <> (SELECT COUNT(*) FROM BookSeed); + +-- ...and every one joins through to a category, which is what the ADFA-5204 +-- breakage failed: rows present, join empty. +INSERT INTO Problems (problem) +SELECT 'only ' || (SELECT COUNT(*) FROM Content C, Bookshelf B, BookCategories BC + WHERE C.id = B.contentID AND B.bookCategoryID = BC.id) + || ' of ' || (SELECT COUNT(*) FROM BookSeed) || ' books join to a category' + WHERE (SELECT COUNT(*) FROM Content C, Bookshelf B, BookCategories BC + WHERE C.id = B.contentID AND B.bookCategoryID = BC.id) <> (SELECT COUNT(*) FROM BookSeed); + +-- Every seeded category exists. +INSERT INTO Problems (problem) +SELECT 'these seeded categories are missing: ' || GROUP_CONCAT(category, '; ') + FROM CategorySeed S + WHERE NOT EXISTS (SELECT 1 FROM BookCategories BC WHERE BC.category = S.category) +HAVING COUNT(*) > 0; + +-- Exactly one template row, of the expected size. +INSERT INTO Problems (problem) +SELECT 'expected one bookshelf template of 1261 bytes, found ' + || (SELECT COUNT(*) FROM Templates WHERE name = 'bookshelf') || ' row(s) of ' + || IFNULL((SELECT LENGTH(content) FROM Templates WHERE name = 'bookshelf'), 0) || ' bytes' + WHERE (SELECT COUNT(*) FROM Templates WHERE name = 'bookshelf') <> 1 + OR (SELECT LENGTH(content) FROM Templates WHERE name = 'bookshelf') <> 1261; + +-- Prints one line per problem; silent on a clean run. +SELECT 'VERIFICATION FAILED: ' || problem FROM Problems; + +-- Any problem makes this insert violate the CHECK, which aborts and rolls back. +CREATE TEMP TABLE Gate ( + ok INTEGER NOT NULL CHECK (ok = 1) +); +INSERT INTO Gate (ok) +SELECT CASE WHEN (SELECT COUNT(*) FROM Problems) = 0 THEN 1 ELSE 0 END; + +DROP TABLE Gate; +DROP TABLE Problems; +DROP TABLE BookSeed; +DROP TABLE CategorySeed; + +COMMIT; + +-- What the result should look like: +-- +-- Android |Android Notes for Professionals +-- C and C++ |Beej Guide To C Programming +-- C and C++ |Modern C++ Tutorial +-- Java |Java Notes for Professionals +-- Java |Java, Java, Java: Object-Oriented Problem Solving +-- Kotlin |Kotlin Notes for Professionals +-- Pebble |Pebble Template Guide diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 566703ad1b..6a2090312c 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -94,10 +94,12 @@ Schema changes and data edits happen **outside this repo**, in `OfflineDocumenta ### Writing one-off SQL scripts against this database -Some tickets (e.g. ADFA-5088) ship a one-off `.sql` script under `docs/docdb/` for a `docdb-studio` maintainer to run against the real database, rather than editing it directly through the tool. Gotchas found writing those scripts: +Some tickets (e.g. ADFA-5088, ADFA-5212) ship a one-off `.sql` script under `docs/docdb/` for a `docdb-studio` maintainer to run against the real database, rather than editing it directly through the tool. Gotchas found writing those scripts: - **Keep each `.system` line simple.** The sqlite3 CLI's `.system` dot-command can hit a content-dependent shell-parsing failure when a line chains multiple operators (`;`, `&&`, `||`, parentheses) — it reproduces for some input strings and not others, so it won't necessarily show up in a quick test. Stick to one plain `command | pipe > file` per `.system` line. - **`.bail on` is required for `BEGIN`/`COMMIT` to actually mean atomic.** Without it, a mid-script SQL error prints to stderr but the script *keeps going* — including reaching the final `COMMIT`, which then persists whatever succeeded before the error (verified empirically, not just documented behavior). `.bail` also can't see `.system` shell failures directly, so a failed or empty Brotli payload (which leaves its target file missing or zero-length) needs its own check: insert its `READFILE()` into a throwaway `CREATE TEMP TABLE` guarded by `NOT NULL CHECK (length(content) > 0)` immediately before the real `Content` insert, turning that failure into a real SQL error `.bail` will catch. See `docs/docdb/ADFA-5088-preference-tooltips.sql` for the working pattern. +- **Never hard-code a `Content.id`.** They're `AUTOINCREMENT` values assigned at import, so they differ in every rebuild of the database — a script that names them silently attaches its rows to whatever content now holds those ids. Resolve by `Content.path` instead (`INSERT INTO … SELECT id, … FROM Content WHERE path = '…'`), which is stable and fails safe: a missing path inserts nothing rather than mislinking. See `docs/docdb/ADFA-5212-dynamic-bookshelf.sql`. +- **`CHECK` can't hold a subquery**, so the ADFA-5088 pattern above doesn't extend to invariants that need to count rows. Collect violations into a temp table with `INSERT … WHERE ` (subqueries are fine there), `SELECT` them so the maintainer sees *which* invariant broke, then gate on `CHECK (ok = 1)` fed by `COUNT(*) = 0`. And give any aggregate check a `HAVING COUNT(*) > 0`: an aggregate with no `GROUP BY` returns one row even when nothing matched, so `GROUP_CONCAT` yields NULL and a `NOT NULL` column then fails the *clean* run. - **Don't write Brotli payloads to bare `/tmp/*.br` filenames.** A fixed, guessable name directly under world-writable `/tmp` lets another local user pre-plant a symlink or race the write/read pair between the `.system echo | brotli` write and the `READFILE()` read (CWE-377). Create an owner-only working directory instead — `rm -rf` it, then `mkdir -m 700` it (the mode is set atomically at creation, with no window where it's briefly world-accessible) — write every payload under that directory, and remove it again before `COMMIT`. See the same script for the working pattern. ## Known rough edges From d36ab8114bf9ef270e7af0b5ccf2a54346a7c02e Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 24 Aug 2026 15:05:44 -0700 Subject: [PATCH 2/4] ADFA-5212: Delete only the bookshelf rows this migration owns DELETE FROM Bookshelf took out plugin-contributed books along with the rows being re-seeded, and the checks afterwards still passed because they counted only the seeded rows they expected to find -- so the loss was silent. The delete is now scoped to rows this script owns: the ones it is about to re-seed, and the ADFA-5204 placeholder rows, which are identifiable by having no category to join to. A plugin row keeps a real bookCategoryID and a contentID outside the seed set, so it survives both clauses. The two count checks are scoped the same way, so a plugin's books neither satisfy them nor break them. Verified against a copy of the real database with a plugin-style row added: the previous version left 7 rows with that row gone and reported nothing; this one leaves 8 with the row intact, twice in a row. Co-Authored-By: Claude Opus 5 --- docs/docdb/ADFA-5212-dynamic-bookshelf.sql | 41 +++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/docs/docdb/ADFA-5212-dynamic-bookshelf.sql b/docs/docdb/ADFA-5212-dynamic-bookshelf.sql index e8f6f2c901..934fd87b6b 100644 --- a/docs/docdb/ADFA-5212-dynamic-bookshelf.sql +++ b/docs/docdb/ADFA-5212-dynamic-bookshelf.sql @@ -144,10 +144,22 @@ UPDATE BookCategories -- --------------------------------------------------------------------- -- Rebuilt rather than merged. The 14-Aug database's rows are unusable (NULL -- bookCategoryID, datetime-placeholder titles; see ADFA-5204), and adding to --- them leaves real books beside placeholder ones. If a database ever carries --- bookshelf rows from another source, narrow this DELETE to the seeded paths. +-- them leaves real books beside placeholder ones. +-- +-- Scoped to what this migration owns, not "everything". A plugin can contribute +-- non-PDF books to this catalog (see docs/documentation-database.md), and those +-- rows are none of this script's business: DELETE FROM Bookshelf would have taken +-- them out and the checks below would still have passed, because they counted only +-- the seeded rows they expected to find. +-- +-- Two things go: rows this migration is about to re-seed, and the ADFA-5204 +-- placeholder rows, which are identifiable by having no category to join to. A +-- plugin row keeps a real bookCategoryID and a contentID outside BookSeed, so it +-- survives both clauses. -DELETE FROM Bookshelf; +DELETE FROM Bookshelf + WHERE bookCategoryID IS NULL + OR contentID IN (SELECT C.id FROM Content C JOIN BookSeed S ON C.path = S.path); INSERT INTO Bookshelf (contentID, title, description, bookCategoryID) SELECT C.id, S.title, S.description, BC.id @@ -201,19 +213,24 @@ SELECT 'these seeded book paths are missing from Content: ' || GROUP_CONCAT(path -- is NULL, and the insert fails where nothing is wrong. HAVING COUNT(*) > 0; --- The bookshelf holds exactly the seeded books. +-- Every seeded book is on the shelf. Counted over the seeded rows rather than the +-- whole table, so a plugin's own books neither satisfy this check nor break it. INSERT INTO Problems (problem) -SELECT 'Bookshelf has ' || (SELECT COUNT(*) FROM Bookshelf) || ' rows, expected ' || (SELECT COUNT(*) FROM BookSeed) - WHERE (SELECT COUNT(*) FROM Bookshelf) <> (SELECT COUNT(*) FROM BookSeed); +SELECT 'Bookshelf holds ' || (SELECT COUNT(*) FROM Bookshelf B, Content C, BookSeed S + WHERE C.id = B.contentID AND C.path = S.path) + || ' of the ' || (SELECT COUNT(*) FROM BookSeed) || ' seeded books' + WHERE (SELECT COUNT(*) FROM Bookshelf B, Content C, BookSeed S + WHERE C.id = B.contentID AND C.path = S.path) <> (SELECT COUNT(*) FROM BookSeed); --- ...and every one joins through to a category, which is what the ADFA-5204 +-- ...and every seeded one joins through to a category, which is what the ADFA-5204 -- breakage failed: rows present, join empty. INSERT INTO Problems (problem) -SELECT 'only ' || (SELECT COUNT(*) FROM Content C, Bookshelf B, BookCategories BC - WHERE C.id = B.contentID AND B.bookCategoryID = BC.id) - || ' of ' || (SELECT COUNT(*) FROM BookSeed) || ' books join to a category' - WHERE (SELECT COUNT(*) FROM Content C, Bookshelf B, BookCategories BC - WHERE C.id = B.contentID AND B.bookCategoryID = BC.id) <> (SELECT COUNT(*) FROM BookSeed); +SELECT 'only ' || (SELECT COUNT(*) FROM Content C, Bookshelf B, BookCategories BC, BookSeed S + WHERE C.id = B.contentID AND B.bookCategoryID = BC.id AND C.path = S.path) + || ' of ' || (SELECT COUNT(*) FROM BookSeed) || ' seeded books join to a category' + WHERE (SELECT COUNT(*) FROM Content C, Bookshelf B, BookCategories BC, BookSeed S + WHERE C.id = B.contentID AND B.bookCategoryID = BC.id AND C.path = S.path) + <> (SELECT COUNT(*) FROM BookSeed); -- Every seeded category exists. INSERT INTO Problems (problem) From 7b32e599d768dc7e3a1ee12e6ba185ae8c7ecd83 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Tue, 25 Aug 2026 17:35:27 -0700 Subject: [PATCH 3/4] ADFA-5212: Render every category, store text as text, keep one template Review of #1707 found three defects the script could not catch itself, because its own checks were narrower than its effects. The template called a filter-by-category macro once per category and named five, while the seed above defines six. General could never appear on the page -- and General is also WebServer's IFNULL fallback name for an uncategorised book. It now loops over the payload, so every category the query returns is rendered, and each row is visited once instead of five times. Display order comes from the query's ORDER BY BC.category rather than the call order; pinning a different order needs a sort column in BookCategories. The seeded descriptions held HTML entities. Pebble auto-escapes on output, so & reached the browser as &amp; and “ as visible markup, in three of the seven books. They are stored as the characters they stand for now; the response already declares utf-8. The 1261-byte template blob appeared twice, verbatim, in the INSERT and the UPDATE, guarded only by a length check -- so any length-neutral edit to one copy would install a different template on the fresh path than on the update path, and verification would pass. It lives in one temp table both statements read, and verification compares content rather than LENGTH() <> 1261, which is what let the escaping bug through. The DELETE loses its bookCategoryID IS NULL clause. It was meant to sweep up ADFA-5204 placeholder rows, but the AddBook trigger writes (contentID, CURRENT_TIMESTAMP || id) and nothing else, so a NULL category and a datetime-looking title are what every freshly ingested PDF has until someone curates it. The placeholder rows and the pending ones are the same rows and no WHERE clause separates them, while DeleteBook only fires on a Content DELETE, so anything removed here never returns. Uncurated rows are now reported by a whole-table check instead -- the ADFA-5204 symptom was a shelf holding rows nobody expected, which a check scoped to the seeded rows cannot see. Also corrects the co-author of Java, Java, Java: Ralph Walde, not Wade. Verified against a copy of a real documentation.db (2026-08-21, 238 MB): the script runs clean and reports no problems, re-running it is also clean, all six categories exist, and the stored template is byte-equal to the intended one. The template was then rendered through Pebble 4.1.1 with WebServer's own engine construction and gson settings, against the payload readBookshelf builds: General renders, there are zero occurrences of &amp; or of literal “/”, the three ampersands are escaped exactly once, and no empty description heading is emitted for a category whose description is null. Found in review of PR #1707. --- docs/docdb/ADFA-5212-dynamic-bookshelf.sql | 78 +++++++++++++++------- 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/docs/docdb/ADFA-5212-dynamic-bookshelf.sql b/docs/docdb/ADFA-5212-dynamic-bookshelf.sql index 934fd87b6b..d8be76b0bb 100644 --- a/docs/docdb/ADFA-5212-dynamic-bookshelf.sql +++ b/docs/docdb/ADFA-5212-dynamic-bookshelf.sql @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS Bookshelf ( -- at the end can compare against what was asked for. -- -- Descriptions are inserted into HTML by the template, so the entities are --- deliberate. The prototype had a bare '&' in the Kotlin entry; it is '&' +-- deliberate. Stored as the characters themselves, not HTML entities: Pebble -- here, matching the others. CREATE TEMP TABLE CategorySeed ( @@ -100,11 +100,11 @@ INSERT INTO BookSeed (path, title, description, category) VALUES 'Android'), ('bookshelf/org.appdevforall.bookshelfplugin/BeejGuideToCProgramming.pdf', 'Beej Guide To C Programming', - 'By Brian “Beej Jorgensen” Hall. 41 chapters, 342 pages covers C, the preprocessor, and the standard library', + 'By Brian “Beej Jorgensen” Hall. 41 chapters, 342 pages covers C, the preprocessor, and the standard library', 'C and C++'), ('bookshelf/org.appdevforall.bookshelfplugin/JavaJavaJava.pdf', 'Java, Java, Java: Object-Oriented Problem Solving', - 'By Ralph Morelli and Ralph Wade. 16 chapters and 8 appendices, 840 pages covers basic Java including exceptions, also graphics, threads, and socket programming', + 'By Ralph Morelli and Ralph Walde. 16 chapters and 8 appendices, 840 pages covers basic Java including exceptions, also graphics, threads, and socket programming', 'Java'), ('bookshelf/org.appdevforall.bookshelfplugin/JavaNotesForProfessionals.pdf', 'Java Notes for Professionals', @@ -112,11 +112,11 @@ INSERT INTO BookSeed (path, title, description, category) VALUES 'Java'), ('bookshelf/org.appdevforall.bookshelfplugin/KotlinNotesForProfessionals.pdf', 'Kotlin Notes for Professionals', - 'Compiled from Stack Overflow. 94 pages, covering Basics & Control Flow, Null Safety, Object-Oriented & Functional Mix, Advanced Features, Java Interoperability, Android Specifics, and more', + 'Compiled from Stack Overflow. 94 pages, covering Basics & Control Flow, Null Safety, Object-Oriented & Functional Mix, Advanced Features, Java Interoperability, Android Specifics, and more', 'Kotlin'), ('bookshelf/org.appdevforall.bookshelfplugin/ModernCplusplusTutorialOuChangkun.pdf', 'Modern C++ Tutorial', - 'By Ou Changkun. 10 chapters and 2 appendices, 111 pages, “C++ programmers who are still using traditional C++ (this book refers to C++98 and its previous standards as traditional C++) may even amazed by the fact that they are not using the same language while reading modern C++ code.”', + 'By Ou Changkun. 10 chapters and 2 appendices, 111 pages, “C++ programmers who are still using traditional C++ (this book refers to C++98 and its previous standards as traditional C++) may even amazed by the fact that they are not using the same language while reading modern C++ code.”', 'C and C++'), ('bookshelf/org.appdevforall.bookshelfplugin/PebbleTemplateGuide.pdf', 'Pebble Template Guide', @@ -152,14 +152,21 @@ UPDATE BookCategories -- them out and the checks below would still have passed, because they counted only -- the seeded rows they expected to find. -- --- Two things go: rows this migration is about to re-seed, and the ADFA-5204 --- placeholder rows, which are identifiable by having no category to join to. A --- plugin row keeps a real bookCategoryID and a contentID outside BookSeed, so it --- survives both clauses. +-- Only rows this migration is about to re-seed. There was a second clause, +-- bookCategoryID IS NULL, meant to sweep up the ADFA-5204 placeholder rows -- but +-- NULL does not mean what it assumed. The AddBook trigger is +-- +-- AFTER INSERT ON Content WHEN NEW.path LIKE '%.pdf' +-- INSERT INTO Bookshelf (contentID, title) VALUES (NEW.id, CURRENT_TIMESTAMP || NEW.id); +-- +-- so it writes contentID and a timestamp title and nothing else. A NULL category and +-- a datetime-looking title are what *every* freshly ingested PDF has until someone +-- curates it -- the placeholder rows and the pending ones are the same rows, and no +-- WHERE clause can separate them. DeleteBook only fires on a Content DELETE, so +-- anything removed here never comes back. DELETE FROM Bookshelf - WHERE bookCategoryID IS NULL - OR contentID IN (SELECT C.id FROM Content C JOIN BookSeed S ON C.path = S.path); + WHERE contentID IN (SELECT C.id FROM Content C JOIN BookSeed S ON C.path = S.path); INSERT INTO Bookshelf (contentID, title, description, bookCategoryID) SELECT C.id, S.title, S.description, BC.id @@ -174,18 +181,27 @@ SELECT C.id, S.title, S.description, BC.id -- is left alone on an update, because Content.templateId refers to template -- ids numerically and reassigning them is a hazard worth avoiding entirely. -- --- The blob is the template verified on device on 19-Aug: 1261 bytes, with the --- debug output removed (it was most of the rendered page) and its five --- category names matching the seed data above. Those names are hardcoded in --- the template on purpose, to control display order without a sort column in --- BookCategories, so a seventh category needs a template edit too. +-- The blob is the template verified on device on 19-Aug, with the debug output +-- removed (it was most of the rendered page), reworked to loop over the payload +-- instead of naming categories. It used to call a filter-by-category macro once +-- per category, and named five while the seed above defines six, so General -- +-- which is also WebServer's IFNULL fallback name for an uncategorised book -- +-- could never appear on the page. Display order now comes from the query's +-- ORDER BY BC.category rather than the call order; pinning a different order +-- needs a sort column in BookCategories, not a template edit. +-- +-- Held in a temp table so the INSERT and the UPDATE cannot drift apart: they +-- used to carry two copies of the literal, and the check at the bottom compared +-- only its length. + +CREATE TEMP TABLE TemplateBlob AS SELECT X'3C21444F43545950452068746D6C3E0A3C68746D6C206C616E673D22656E2D7573223E0A3C686561643E0A3C7374796C6520747970653D22746578742F637373223E0A2E66696C656C696E6B207B206261636B67726F756E642D636F6C6F723A2079656C6C6F773B207D0A2E7765626C696E6B207B20206261636B67726F756E642D636F6C6F723A206379616E3B7D0A3C2F7374796C653E0A3C2F686561643E3C626F64793E3C703E54686520666F6C6C6F77696E6720626F6F6B7320616E64207265666572656E6365206D6174657269616C732061726520696E636C75646564207769746820436F6465206F6E2074686520476F2E3C2F703E0A7B2520666F72206974656D20696E20726573756C7420257D0A3C68313E43617465676F72793A207B7B206974656D2E63617465676F7279207D7D3C2F68313E0A7B25206966206974656D2E6465736372697074696F6E20257D0A3C68323E4465736372697074696F6E3A207B7B206974656D2E6465736372697074696F6E207D7D3C2F68323E0A7B2520656E64696620257D0A20207B2520666F7220626F6F6B20696E206974656D2E626F6F6B7320257D0A3C703E0A202020207B2520696620626F6F6B2E706466203D3D203120257D0A20203C6120687265663D222F702F7765622F7669657765722E68746D6C3F66696C653D2F7B7B20626F6F6B2E6C696E6B207D7D22207461726765743D225F626C616E6B2220636C6173733D2266696C656C696E6B223E7B7B20626F6F6B2E7469746C65207D7D3C2F613E0A2020202020207B2520696620626F6F6B2E6465736372697074696F6E20257D0A2020202020202020287B7B20626F6F6B2E6465736372697074696F6E207D7D203C693E5044463C2F693E290A2020202020207B2520656E64696620257D0A202020207B2520656C736520257D0A20203C6120687265663D222F7B7B20626F6F6B2E6C696E6B207D7D22207461726765743D225F626C616E6B2220636C6173733D227765626C696E6B223E7B7B20626F6F6B2E7469746C65207D7D3C2F613E0A2020202020207B2520696620626F6F6B2E6465736372697074696F6E20257D0A2020202020202020287B7B20626F6F6B2E6465736372697074696F6E207D7D290A2020202020207B2520656E64696620257D0A202020207B2520656E64696620257D0A3C2F703E0A20207B2520656E64666F7220257D0A7B2520656E64666F7220257D0A3C2F626F64793E3C2F68746D6C3E0A' AS content; INSERT INTO Templates (name, content) -SELECT 'bookshelf', X'3c21444f43545950452068746d6c3e0a3c68746d6c206c616e673d22656e2d7573223e0a3c686561643e0a3c7374796c6520747970653d22746578742f637373223e0a2e66696c656c696e6b207b206261636b67726f756e642d636f6c6f723a2079656c6c6f773b207d0a2e7765626c696e6b207b20206261636b67726f756e642d636f6c6f723a206379616e3b7d0a3c2f7374796c653e0a7b25206d6163726f20657870616e64426f6f6b7328726573756c742c2063617465676f72792920257d0a20207b2520666f72206974656d20696e20726573756c7420257d0a202020207b25206966206974656d2e63617465676f7279203d3d2063617465676f727920257d0a3c68313e43617465676f72793a207b7b2063617465676f7279207d7d3c2f68313e0a7b25206966206974656d2e6465736372697074696f6e20213d20272720257d0a3c68323e4465736372697074696f6e3a207b7b206974656d2e6465736372697074696f6e207d7d3c2f68323e0a7b2520656e64696620257d0a2020202020207b2520666f7220626f6f6b20696e206974656d2e626f6f6b7320257d0a3c703e0a20202020202020207b2520696620626f6f6b2e706466203d3d203120257d0a20203c6120687265663d222f702f7765622f7669657765722e68746d6c3f66696c653d2f7b7b20626f6f6b2e6c696e6b207d7d22207461726765743d225f626c616e6b2220636c6173733d2266696c656c696e6b223e7b7b20626f6f6b2e7469746c65207d7d3c2f613e0a202020202020202020207b2520696620626f6f6b2e6465736372697074696f6e20213d20272720257d0a202020202020202020202020287b7b20626f6f6b2e6465736372697074696f6e207d7d203c693e5044463c2f693e290a202020202020202020207b2520656e64696620257d0a20202020202020207b2520656c736520257d0a20203c6120687265663d222f7b7b20626f6f6b2e6c696e6b207d7d22207461726765743d225f626c616e6b2220636c6173733d227765626c696e6b223e7b7b20626f6f6b2e7469746c65207d7d3c2f613e0a202020202020202020207b2520696620626f6f6b2e6465736372697074696f6e20213d20272720257d0a202020202020202020202020287b7b20626f6f6b2e6465736372697074696f6e207d7d290a202020202020202020207b2520656e64696620257d0a20202020202020207b2520656e64696620257d0a3c2f703e0a2020202020207b2520656e64666f7220257d0a202020207b2520656e64696620257d0a20207b2520656e64666f7220257d0a7b2520656e646d6163726f20257d0a0a3c2f686561643e3c626f64793e3c703e54686520666f6c6c6f77696e6720626f6f6b7320616e64207265666572656e6365206d6174657269616c732061726520696e636c75646564207769746820436f6465206f6e2074686520476f2e3c2f703e0a7b7b20657870616e64426f6f6b7328726573756c742c2027416e64726f69642729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274a6176612729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274b6f746c696e2729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274320616e6420432b2b2729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c2027506562626c652729207d7d0a3c2f626f64793e3c2f68746d6c3e0a' +SELECT 'bookshelf', (SELECT content FROM TemplateBlob) WHERE NOT EXISTS (SELECT 1 FROM Templates WHERE name = 'bookshelf'); UPDATE Templates - SET content = X'3c21444f43545950452068746d6c3e0a3c68746d6c206c616e673d22656e2d7573223e0a3c686561643e0a3c7374796c6520747970653d22746578742f637373223e0a2e66696c656c696e6b207b206261636b67726f756e642d636f6c6f723a2079656c6c6f773b207d0a2e7765626c696e6b207b20206261636b67726f756e642d636f6c6f723a206379616e3b7d0a3c2f7374796c653e0a7b25206d6163726f20657870616e64426f6f6b7328726573756c742c2063617465676f72792920257d0a20207b2520666f72206974656d20696e20726573756c7420257d0a202020207b25206966206974656d2e63617465676f7279203d3d2063617465676f727920257d0a3c68313e43617465676f72793a207b7b2063617465676f7279207d7d3c2f68313e0a7b25206966206974656d2e6465736372697074696f6e20213d20272720257d0a3c68323e4465736372697074696f6e3a207b7b206974656d2e6465736372697074696f6e207d7d3c2f68323e0a7b2520656e64696620257d0a2020202020207b2520666f7220626f6f6b20696e206974656d2e626f6f6b7320257d0a3c703e0a20202020202020207b2520696620626f6f6b2e706466203d3d203120257d0a20203c6120687265663d222f702f7765622f7669657765722e68746d6c3f66696c653d2f7b7b20626f6f6b2e6c696e6b207d7d22207461726765743d225f626c616e6b2220636c6173733d2266696c656c696e6b223e7b7b20626f6f6b2e7469746c65207d7d3c2f613e0a202020202020202020207b2520696620626f6f6b2e6465736372697074696f6e20213d20272720257d0a202020202020202020202020287b7b20626f6f6b2e6465736372697074696f6e207d7d203c693e5044463c2f693e290a202020202020202020207b2520656e64696620257d0a20202020202020207b2520656c736520257d0a20203c6120687265663d222f7b7b20626f6f6b2e6c696e6b207d7d22207461726765743d225f626c616e6b2220636c6173733d227765626c696e6b223e7b7b20626f6f6b2e7469746c65207d7d3c2f613e0a202020202020202020207b2520696620626f6f6b2e6465736372697074696f6e20213d20272720257d0a202020202020202020202020287b7b20626f6f6b2e6465736372697074696f6e207d7d290a202020202020202020207b2520656e64696620257d0a20202020202020207b2520656e64696620257d0a3c2f703e0a2020202020207b2520656e64666f7220257d0a202020207b2520656e64696620257d0a20207b2520656e64666f7220257d0a7b2520656e646d6163726f20257d0a0a3c2f686561643e3c626f64793e3c703e54686520666f6c6c6f77696e6720626f6f6b7320616e64207265666572656e6365206d6174657269616c732061726520696e636c75646564207769746820436f6465206f6e2074686520476f2e3c2f703e0a7b7b20657870616e64426f6f6b7328726573756c742c2027416e64726f69642729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274a6176612729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274b6f746c696e2729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c20274320616e6420432b2b2729207d7d0a7b7b20657870616e64426f6f6b7328726573756c742c2027506562626c652729207d7d0a3c2f626f64793e3c2f68746d6c3e0a' + SET content = (SELECT content FROM TemplateBlob) WHERE name = 'bookshelf'; -- --------------------------------------------------------------------- @@ -232,6 +248,17 @@ SELECT 'only ' || (SELECT COUNT(*) FROM Content C, Bookshelf B, BookCategories B WHERE C.id = B.contentID AND B.bookCategoryID = BC.id AND C.path = S.path) <> (SELECT COUNT(*) FROM BookSeed); +-- Nothing else is on the shelf that this script did not put there or knowingly +-- leave alone. The ADFA-5204 symptom was a shelf holding rows nobody expected, and +-- a check scoped to the seeded rows cannot see those at all. Uncurated books are +-- reported, not deleted: they are pending curation, not junk (see the DELETE above). +INSERT INTO Problems (problem) +SELECT 'note: Bookshelf holds ' || (SELECT COUNT(*) FROM Bookshelf) || ' rows, of which ' + || (SELECT COUNT(*) FROM BookSeed) || ' are seeded here and ' + || (SELECT COUNT(*) FROM Bookshelf WHERE bookCategoryID IS NULL) + || ' are uncurated (no category, so they render under General)' + WHERE (SELECT COUNT(*) FROM Bookshelf) <> (SELECT COUNT(*) FROM BookSeed); + -- Every seeded category exists. INSERT INTO Problems (problem) SELECT 'these seeded categories are missing: ' || GROUP_CONCAT(category, '; ') @@ -239,13 +266,18 @@ SELECT 'these seeded categories are missing: ' || GROUP_CONCAT(category, '; ') WHERE NOT EXISTS (SELECT 1 FROM BookCategories BC WHERE BC.category = S.category) HAVING COUNT(*) > 0; --- Exactly one template row, of the expected size. +-- Exactly one template row, holding exactly this template. Compared by content: +-- a length check passes a wrong template of the right size, which is how the +-- double-escaped descriptions shipped in the first place. INSERT INTO Problems (problem) -SELECT 'expected one bookshelf template of 1261 bytes, found ' - || (SELECT COUNT(*) FROM Templates WHERE name = 'bookshelf') || ' row(s) of ' - || IFNULL((SELECT LENGTH(content) FROM Templates WHERE name = 'bookshelf'), 0) || ' bytes' +SELECT 'expected one bookshelf template matching this script, found ' + || (SELECT COUNT(*) FROM Templates WHERE name = 'bookshelf') || ' row(s), ' + || IFNULL((SELECT LENGTH(content) FROM Templates WHERE name = 'bookshelf'), 0) + || ' bytes against the expected ' + || (SELECT LENGTH(content) FROM TemplateBlob) || ' bytes' WHERE (SELECT COUNT(*) FROM Templates WHERE name = 'bookshelf') <> 1 - OR (SELECT LENGTH(content) FROM Templates WHERE name = 'bookshelf') <> 1261; + OR (SELECT content FROM Templates WHERE name = 'bookshelf') + IS NOT (SELECT content FROM TemplateBlob); -- Prints one line per problem; silent on a clean run. SELECT 'VERIFICATION FAILED: ' || problem FROM Problems; From 47228e20ab6548021f6c71b5994bdb7bde14489d Mon Sep 17 00:00:00 2001 From: David Schachter Date: Wed, 26 Aug 2026 14:56:46 -0700 Subject: [PATCH 4/4] ADFA-5212: Stop the informational note from rolling the migration back The whole-table row-count check I added wrote its result into Problems. Problems is the abort list -- the Gate rolls the entire transaction back if it holds anything -- so an informational note made this script a guaranteed no-op on every database with an uncurated PDF. Which is every real database: the AddBook trigger creates exactly that state for each newly ingested book, as the comment a hundred lines above says. Reproduced on a synthetic database with the 7 seeded books plus one other PDF: "VERIFICATION FAILED: note: Bookshelf holds 8 rows...", CHECK constraint failed, rollback, BookCategories left at 0 and no template installed. It now prints instead of inserting, and the same fixture applies cleanly: 6 categories, 7 seeded books, the uncurated row untouched. TemplateBlob was the one temp table nothing dropped, so a second .read in the same sqlite3 session died on "table TemplateBlob already exists" -- against a script whose header promises running it twice is safe. Two runs in one session now give identical results. The template content check was tautological. It compared the stored template against TemplateBlob, two statements after the UPDATE that sets the stored template FROM TemplateBlob, so it could not fail whatever blob the script carried -- and my commit message claimed it caught "a wrong template of the right size". It checks the length against a literal instead, which can fail: a row the UPDATE did not match, or one another tool installed. It still cannot tell two different 905-byte templates apart, and now says so. Also corrected a claim about General. WebServer's inner join (B.bookCategoryID = BC.id) drops uncategorised books before IFNULL(BC.category, 'General') can label them, so that fallback only fires for a category row whose own name is NULL -- an uncurated book does not "render under General", it does not render at all. Found in review of PR #1707. --- docs/docdb/ADFA-5212-dynamic-bookshelf.sql | 40 ++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/docs/docdb/ADFA-5212-dynamic-bookshelf.sql b/docs/docdb/ADFA-5212-dynamic-bookshelf.sql index d8be76b0bb..928745e66f 100644 --- a/docs/docdb/ADFA-5212-dynamic-bookshelf.sql +++ b/docs/docdb/ADFA-5212-dynamic-bookshelf.sql @@ -185,7 +185,7 @@ SELECT C.id, S.title, S.description, BC.id -- removed (it was most of the rendered page), reworked to loop over the payload -- instead of naming categories. It used to call a filter-by-category macro once -- per category, and named five while the seed above defines six, so General -- --- which is also WebServer's IFNULL fallback name for an uncategorised book -- +-- which WebServer also uses as its IFNULL label for a category row whose own name is NULL -- -- could never appear on the page. Display order now comes from the query's -- ORDER BY BC.category rather than the call order; pinning a different order -- needs a sort column in BookCategories, not a template edit. @@ -248,15 +248,40 @@ SELECT 'only ' || (SELECT COUNT(*) FROM Content C, Bookshelf B, BookCategories B WHERE C.id = B.contentID AND B.bookCategoryID = BC.id AND C.path = S.path) <> (SELECT COUNT(*) FROM BookSeed); --- Nothing else is on the shelf that this script did not put there or knowingly --- leave alone. The ADFA-5204 symptom was a shelf holding rows nobody expected, and --- a check scoped to the seeded rows cannot see those at all. Uncurated books are --- reported, not deleted: they are pending curation, not junk (see the DELETE above). +-- Every seeded category exists. INSERT INTO Problems (problem) +SELECT 'these seeded categories are missing: ' || GROUP_CONCAT(category, '; ') + FROM CategorySeed S + WHERE NOT EXISTS (SELECT 1 FROM BookCategories BC WHERE BC.category = S.category) +HAVING COUNT(*) > 0; + +-- Exactly one template row, of exactly this template's length. +-- +-- Length, not content: the UPDATE above sets content FROM TemplateBlob, so comparing the stored +-- value against TemplateBlob is tautological -- it cannot fail, whatever blob this script carries. +-- A literal length can fail, and catches the case worth catching: a row this script's UPDATE did +-- not touch (a name mismatch) or one another tool installed. It cannot tell two different 905-byte +-- templates apart; nothing available in sqlite3 can, short of shipping a checksum. +INSERT INTO Problems (problem) +SELECT 'expected one bookshelf template of 905 bytes, found ' + || (SELECT COUNT(*) FROM Templates WHERE name = 'bookshelf') || ' row(s) of ' + || IFNULL((SELECT LENGTH(content) FROM Templates WHERE name = 'bookshelf'), 0) || ' bytes' + WHERE (SELECT COUNT(*) FROM Templates WHERE name = 'bookshelf') <> 1 + OR IFNULL((SELECT LENGTH(content) FROM Templates WHERE name = 'bookshelf'), 0) <> 905; + +-- Nothing else is on the shelf that this script did not put there or knowingly left alone. The +-- ADFA-5204 symptom was a shelf holding rows nobody expected, and a check scoped to the seeded rows +-- cannot see those at all. +-- +-- Printed, NOT inserted into Problems. Problems is the abort list: the Gate below rolls the whole +-- migration back if it holds anything, so writing an informational row there made this script a +-- guaranteed no-op on every database with an uncurated PDF -- which is every database, since the +-- AddBook trigger creates exactly that state for each newly ingested book (see the DELETE above). +-- Verified: 7 seeded books plus one other PDF rolled back with BookCategories left at 0. SELECT 'note: Bookshelf holds ' || (SELECT COUNT(*) FROM Bookshelf) || ' rows, of which ' || (SELECT COUNT(*) FROM BookSeed) || ' are seeded here and ' || (SELECT COUNT(*) FROM Bookshelf WHERE bookCategoryID IS NULL) - || ' are uncurated (no category, so they render under General)' + || ' await curation (no category, so WebServer''s inner join leaves them off the page)' WHERE (SELECT COUNT(*) FROM Bookshelf) <> (SELECT COUNT(*) FROM BookSeed); -- Every seeded category exists. @@ -290,6 +315,9 @@ INSERT INTO Gate (ok) SELECT CASE WHEN (SELECT COUNT(*) FROM Problems) = 0 THEN 1 ELSE 0 END; DROP TABLE Gate; +-- TemplateBlob too, or a second .read in the same sqlite3 session fails on "table +-- TemplateBlob already exists" -- which contradicts this script's own idempotency claim. +DROP TABLE TemplateBlob; DROP TABLE Problems; DROP TABLE BookSeed; DROP TABLE CategorySeed;