diff --git a/.github/workflows/governance.yml b/.github/workflows/governance.yml index 331153a..57efc3b 100644 --- a/.github/workflows/governance.yml +++ b/.github/workflows/governance.yml @@ -26,7 +26,7 @@ jobs: - os: windows-latest runner: windows-latest runs-on: ${{ matrix.runner }} - timeout-minutes: 10 + timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: @@ -35,7 +35,67 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' + - name: Provision the current WFL nightly + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + # The GitHub daily release is an immutable mirror. The canonical + # publisher can release a newer runtime on the same day to this CDN. + $cdn = 'https://wfl.nyc3.cdn.digitaloceanspaces.com' + # A previously cached pointer can retain the CDN's older one-hour TTL. + $publicationUri = "$cdn/status.json?request=$([guid]::NewGuid().ToString('N'))" + $publication = Invoke-RestMethod -Uri $publicationUri + if ($publication.result -ne 'success' -or $publication.branch -ne 'main' -or + $publication.version -notmatch '^\d+\.\d+\.\d+$' -or + $publication.sha -notmatch '^[a-f0-9]{40}$') { + throw 'Invalid official WFL publication record' + } + $releaseVersion = [regex]::Escape($publication.version) + $pattern = if ($IsWindows) { "^wfl-$releaseVersion\.msi$" } else { "^wfl-$releaseVersion-linux-x86_64-[a-f0-9]{7,40}\.tar\.gz$" } + $assets = @($publication.message -split '\s+' | Where-Object { $_ -match $pattern }) + if ($assets.Count -ne 1) { throw 'Publication must name one immutable platform-specific WFL asset' } + $assetName = $assets[0] + if (-not $IsWindows -and $assetName -match '-linux-x86_64-([a-f0-9]{7,40})\.tar\.gz$') { + if (-not $publication.sha.StartsWith($Matches[1])) { throw 'WFL artifact revision does not match publication' } + } + $assetUrl = "$cdn/releases/$assetName" + $checksum = (Invoke-WebRequest -Uri "$assetUrl.sha256").Content.Trim() + $checksumPattern = '^([a-fA-F0-9]{64})\s+\*?' + [regex]::Escape($assetName) + '$' + if ($checksum -notmatch $checksumPattern) { throw 'Invalid immutable WFL checksum sidecar' } + $expectedDigest = $Matches[1].ToLowerInvariant() + $runtimeRoot = Join-Path $env:RUNNER_TEMP 'scriptorium-wfl' + New-Item -ItemType Directory -Path $runtimeRoot -Force | Out-Null + $archive = Join-Path $runtimeRoot $assetName + Invoke-WebRequest -Uri $assetUrl -OutFile $archive + $digest = (Get-FileHash -LiteralPath $archive -Algorithm SHA256).Hash.ToLowerInvariant() + if ($expectedDigest -ne $digest) { throw 'WFL release asset digest mismatch' } + if ($IsWindows) { + $expanded = Join-Path $runtimeRoot 'expanded' + $arguments = @('/a', "`"$archive`"", '/qn', "TARGETDIR=`"$expanded`"") + $installer = Start-Process msiexec.exe -ArgumentList $arguments -Wait -PassThru -WindowStyle Hidden + if ($installer.ExitCode -ne 0) { throw "WFL extraction failed: $($installer.ExitCode)" } + $programs = @(Get-ChildItem -LiteralPath $expanded -Filter wfl.exe -File -Recurse) + } else { + tar -xzf $archive -C $runtimeRoot + if ($LASTEXITCODE -ne 0) { throw 'WFL extraction failed' } + $programs = @(Get-ChildItem -LiteralPath $runtimeRoot -Filter wfl -File -Recurse) + } + if ($programs.Count -ne 1) { throw 'Expected one extracted WFL executable' } + $runtimePath = $programs[0].FullName + $programs[0].DirectoryName | Out-File -FilePath $env:GITHUB_PATH -Append + $version = & $runtimePath --version + if ($LASTEXITCODE -ne 0) { throw 'Extracted WFL runtime did not start' } + if ($version -ne "WebFirst Language (WFL) version $($publication.version)") { throw 'WFL executable version does not match publication' } + @( + '### Governance runtime', + "- Publication: $cdn/status.json", + "- WFL revision: $($publication.sha)", + "- Asset: $assetUrl", + "- SHA256: $digest", + "- Runtime: $version", + "- Scriptorium: $env:GITHUB_SHA" + ) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append - name: Test repository tooling - run: python -m unittest discover -s tests/tooling -v + run: wfl scripts/run_tests.wfl --group tooling - name: Check repository hygiene run: python scripts/check_repo_hygiene.py diff --git a/.github/workflows/runtime-capabilities.yml b/.github/workflows/runtime-capabilities.yml new file mode 100644 index 0000000..8287414 --- /dev/null +++ b/.github/workflows/runtime-capabilities.yml @@ -0,0 +1,92 @@ +name: WFL ORM capability gates + +on: + push: + branches: [codex/wfl-orm-migrations] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: orm-capabilities-${{ github.ref }} + cancel-in-progress: true + +jobs: + runtime: + name: Resolve current nightly + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + image: ${{ steps.runtime.outputs.image }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + - name: Pull nightly and record provenance + id: runtime + shell: bash + run: | + docker pull bsbyrdwfl/wfl:nightly + image="$(docker image inspect --format '{{index .RepoDigests 0}}' bsbyrdwfl/wfl:nightly)" + echo "image=$image" >> "$GITHUB_OUTPUT" + version="$(docker run --rm --network none "$image" --version)" + { + echo '### ORM prerequisite evidence (not completed ORM validation)' + echo "- Image: $image" + echo "- Runtime: $version" + echo "- Scriptorium: $(git rev-parse HEAD)" + echo "- Scribe: $(git -C lib/scribe rev-parse HEAD)" + echo '- Every probe, fixture, HTTP driver and assertion is WFL.' + echo '- Unmet capability assertions fail their jobs without suppression.' + } | tee -a "$GITHUB_STEP_SUMMARY" + + probes: + name: ${{ matrix.suite }} + needs: runtime + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + include: + - suite: orm-native-baseline + path: tests/runtime/orm-native-baseline.test.wfl + - suite: transaction-validation-capability + path: tests/runtime/transaction-validation-capability.test.wfl + - suite: rebuild-foreign-keys-capability + path: tests/runtime/rebuild-foreign-keys-capability.test.wfl + - suite: process-capabilities + path: tests/runtime/process-capabilities.test.wfl + - suite: http-redirect-capability + path: tests/runtime/http-redirect-capability.test.wfl + - suite: media-body-limit + path: tests/integration/media.test.wfl + env: + RESOLVED_IMAGE: ${{ needs.runtime.outputs.image }} + PROBE_SUITE: ${{ matrix.path }} + TEST_CONTAINER: scriptorium-probe-${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + - name: Invoke WFL capability suite + shell: bash + run: | + docker pull "$RESOLVED_IMAGE" + docker run --rm --init --name "$TEST_CONTAINER" \ + --user 0:0 --entrypoint /bin/sh \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/source,readonly" \ + --env PROBE_SUITE --workdir /work "$RESOLVED_IMAGE" -ec ' + cp -a /source/. /work/ + exec wfl --test "$PROBE_SUITE" + ' + - name: Clean up disposable container + if: always() + shell: bash + run: | + if docker container inspect "$TEST_CONTAINER" >/dev/null 2>&1; then + docker container rm --force "$TEST_CONTAINER" + fi diff --git a/.github/workflows/update-scribe.yml b/.github/workflows/update-scribe.yml index dfdb0e0..5388540 100644 --- a/.github/workflows/update-scribe.yml +++ b/.github/workflows/update-scribe.yml @@ -136,8 +136,8 @@ jobs: echo echo '| Check or exact command | Result and evidence |' echo '|---|---|' - echo '| `python3 scripts/run_tests.py --include-scribe` | Pending — this workflow prepares the dependency bump. Link WFL tests results for the current revision, including the nightly image digest and runtime version; record failures. |' - echo '| `python -m unittest discover -s tests/tooling -v` | Pending — Governance results are not verified by this workflow. Link results for the current revision. |' + echo '| `wfl --execution-timeout 1200 scripts/run_tests.wfl` | Pending — this workflow prepares the dependency bump. Link WFL tests results for the current revision, including the nightly image digest and runtime version; record failures. |' + echo '| `wfl scripts/run_tests.wfl --group tooling` | Pending — Governance results are not verified by this workflow. Link results for the current revision. |' echo '| `python scripts/check_repo_hygiene.py` | Pending — Governance results are not verified by this workflow. Link results for the current revision. |' echo '| Affected HTTP/UI rendering journeys | Not run — upstream diff review is needed to identify affected paths. Record setup, expected/actual outcome, and evidence. |' echo diff --git a/.github/workflows/wfl-tests.yml b/.github/workflows/wfl-tests.yml index 4c337f5..2046718 100644 --- a/.github/workflows/wfl-tests.yml +++ b/.github/workflows/wfl-tests.yml @@ -18,7 +18,7 @@ jobs: wfl-tests: name: WFL nightly (Blacksmith) runs-on: blacksmith-2vcpu-ubuntu-2404 - timeout-minutes: 15 + timeout-minutes: 30 env: WFL_IMAGE: bsbyrdwfl/wfl:nightly TEST_CONTAINER: scriptorium-tests-${{ github.run_id }}-${{ github.run_attempt }} @@ -36,36 +36,40 @@ jobs: image="$(docker image inspect --format '{{index .RepoDigests 0}}' "$WFL_IMAGE")" echo "image=$image" >> "$GITHUB_OUTPUT" version="$(docker run --rm --network none "$image" --version)" + runtime_revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")" + [[ "$runtime_revision" =~ ^[a-f0-9]{40}$ ]] || { echo 'Official image is missing its WFL source revision'; exit 1; } { echo '### WFL nightly test environment' echo echo "- Image: $image" echo "- Runtime: $version" + echo "- WFL revision: $runtime_revision" echo "- Scriptorium: $(git rev-parse HEAD)" echo "- Scribe: $(git -C lib/scribe rev-parse HEAD)" echo '- Runner: blacksmith-2vcpu-ubuntu-2404 (Linux x64)' - echo '- Command: python3 scripts/run_tests.py --include-scribe' - echo '- HTTP checks: python3 -m unittest discover -s tests/integration -v' + echo '- Command: wfl --execution-timeout 1200 scripts/run_tests.wfl' + echo '- Coverage: application, ORM/migrations/recovery, HTTP integration, tooling, examples, pinned Scribe' } | tee -a "$GITHUB_STEP_SUMMARY" - - name: Run WFL suites and HTTP configuration checks + - name: Run the complete WFL suite shell: bash env: RESOLVED_IMAGE: ${{ steps.runtime.outputs.image }} run: | # The published image is a minimal runtime with a WFL entrypoint. - # Install Python only in this disposable container. Scribe fixtures - # go to its temporary directory; the checkout stays read-only. + # Python is only the implementation of the repository hygiene checker. + # Every scenario, fixture, assertion and test driver is WFL. A writable + # disposable copy keeps synthetic databases/Git indexes off the checkout. docker run --rm --init --name "$TEST_CONTAINER" \ --user 0:0 --entrypoint /bin/sh \ - --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/source,readonly" \ --workdir /work "$RESOLVED_IMAGE" -ec ' apt-get update - apt-get install --yes --no-install-recommends python3 + apt-get install --yes --no-install-recommends python3 python-is-python3 git + cp -a /source/. /work/ python3 --version wfl --version - python3 scripts/run_tests.py --include-scribe - python3 -m unittest discover -s tests/integration -v + wfl --execution-timeout 1200 scripts/run_tests.wfl ' - name: Clean up test container diff --git a/.repo-hygiene.toml b/.repo-hygiene.toml index 4c4e862..8ab75ed 100644 --- a/.repo-hygiene.toml +++ b/.repo-hygiene.toml @@ -10,7 +10,7 @@ allowed-files = [ "REPOSITORY_HYGIENE.md", "SECURITY.md", "main.wfl", "testing.md", ] allowed-dirs = [ - ".github", "admin", "app", "docs", "lib", "scripts", "static", + ".github", "admin", "app", "docs", "examples", "lib", "scripts", "static", "TestPrograms", "tests", "themes", ] @@ -21,8 +21,9 @@ files = [ "CONTRIBUTING.md", "GOVERNANCE.md", "LICENSE", "README.md", "REPOSITORY_HYGIENE.md", "SECURITY.md", "main.wfl", "testing.md", "docs/ARCHITECTURE.md", "docs/PROJECT-LAYOUT.md", "docs/THEMING.md", - "scripts/check_repo_hygiene.py", "scripts/run_tests.py", - "tests/tooling/test_repo_hygiene.py", "tests/tooling/test_run_tests.py", + "scripts/check_repo_hygiene.py", "scripts/run_tests.wfl", + "scripts/test_support.wfl", "scripts/resolve_runtime.wfl", "scripts/.wflcfg", + "tests/tooling/hygiene.test.wfl", "tests/tooling/runner.test.wfl", "tests/tooling/.wflcfg", ".github/pull_request_template.md", ".github/workflows/governance.yml", ] # These paths must remain Git gitlinks; their contents are upstream-owned. diff --git a/.wflcfg b/.wflcfg index 34138cb..8e3e534 100644 --- a/.wflcfg +++ b/.wflcfg @@ -1,6 +1,6 @@ # Scriptorium configuration (WFL runtime and application settings) timeout_seconds = 60 -logging_enabled = false +execution_logging = false debug_report_enabled = false log_level = info diff --git a/CLAUDE.md b/CLAUDE.md index 79e71d5..aa90ac9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,14 +74,17 @@ violate the standard. - **Scribe is a submodule.** Don't edit `lib/scribe/` in place; changes go upstream to WebFirstLanguage/Scribe, then bump via `scripts/update-scribe.sh`. -- **Checks:** `python scripts/run_tests.py` runs every Scriptorium WFL suite; - add `--include-scribe` for dependency updates. Run - `python -m unittest discover -s tests/tooling -v` and - `python scripts/check_repo_hygiene.py` for repository tooling and hygiene. - Python 3.11+ is needed for tooling; `wfl` is needed for application tests. - The Governance workflow runs tooling tests and hygiene on Blacksmith Linux - and GitHub-hosted Windows. The WFL tests workflow runs the application and - pinned Scribe suites on Blacksmith using a freshly pulled `bsbyrdwfl/wfl:nightly` image; +- **Checks:** `wfl --execution-timeout 1200 scripts/run_tests.wfl` runs the complete suite: application, + ORM/migrations/recovery, HTTP integration, tooling, executable examples, and + pinned Scribe. `--group tooling` or `--group application` selects a focused + run. Run `python scripts/check_repo_hygiene.py` for the repository hygiene gate. + Every test, fixture, assertion, helper and driver is WFL. Python 3.11+ and Git + are required only for the hygiene checker's implementation subject. The runner + uses the WFL executable that launched it, with an optional `--wfl` override. + It needs the owned-process completion and `current_executable` runtime APIs. + Governance provisions WFL and runs tooling and hygiene on Blacksmith Linux + and GitHub-hosted Windows. WFL tests runs the complete suite on Blacksmith + using a freshly pulled `bsbyrdwfl/wfl:nightly` image; its summary records the resolved image digest, runtime version, and source revisions. See [testing.md](testing.md) for commands, coverage limits, and merge evidence. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa1c9a5..01e679b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ the baseline suites: ```sh wfl --version -python scripts/run_tests.py +wfl --execution-timeout 1200 scripts/run_tests.wfl ``` Use `--wfl /absolute/path/to/wfl` if the interpreter is not on `PATH`; use @@ -106,17 +106,16 @@ the PR: ```sh python scripts/check_repo_hygiene.py -python -m unittest discover -s tests/tooling -v -python scripts/run_tests.py +wfl --execution-timeout 1200 scripts/run_tests.wfl ``` -For a Scribe pin, rendering, or template-engine integration change, also run: +The complete command includes pinned Scribe. For a focused Scribe check, run: ```sh -python scripts/run_tests.py --include-scribe +wfl scripts/run_tests.wfl --group scribe ``` -The extra suite runs the pinned Scribe tests in a temporary copy because they +The Scribe group runs the pinned tests in a temporary copy because they write fixtures. Follow [testing.md](testing.md) for HTTP, UI, security, migration, and recovery checks triggered by the change. Prose-only changes need relevant link, command, and hygiene checks; they do not need invented application tests. diff --git a/README.md b/README.md index c511891..5d1c4c2 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,11 @@ engine, and is styled with the **WFL Design System** (dark, teal-on-Ink). - **Swappable themes** — the public site is built from reusable **sections** and assembled into pages: every page is a **header** + a **body** + a **footer**, in that order (`themes/base/`). See [`docs/THEMING.md`](docs/THEMING.md). +- **Reusable WFL ORM and versioned SQLite migrations** — validated models, + records, composed queries, explicit relationship loading and transactional + upgrades for all seven application tables. See the [ORM API](docs/orm.md), + [executable progression](examples/orm/progression.test.wfl), and + [migration and recovery guide](docs/migrations.md). | Admin dashboard | Post editor | Sign in | |---|---|---| @@ -44,7 +49,14 @@ engine, and is styled with the **WFL Design System** (dark, teal-on-Ink). ## Quick start -You need the WFL interpreter (`wfl`) on your PATH. Scriptorium keeps the +You need the WFL interpreter (`wfl`) on your PATH. The verified official runtime +is nightly **26.9.16** from source `23c1a457` (or a newer runtime retaining its +application-error, schema-transaction, HTTP, owned-process and finite invocation-budget capabilities). +Official 26.9.12 lacks those prerequisites. See the +[runtime verification record](docs/orm-verification.md) for immutable provenance. +The complete test command uses the published `--execution-timeout` option to +give the runner a finite 20-minute budget while preserving per-suite deadlines. +Scriptorium keeps the [Scribe](https://github.com/WebFirstLanguage/Scribe) template engine as a git submodule, so clone with submodules: @@ -61,7 +73,8 @@ the working directory): wfl main.wfl ``` -On first run Scriptorium creates `scriptorium.db`, seeds default settings, and +On first run Scriptorium creates `scriptorium.db` through versioned migrations, +seeds default settings, and locks the site behind a **one-page installer**. The console does not print a password: @@ -211,19 +224,19 @@ docs/ Architecture notes + THEMING.md + PROJECT-LAYOUT.md + scre ## Tests -From the repository root, with Python 3.11+ and WFL on PATH: +From the repository root, with WFL, Git and Python 3.11+ on PATH (Python is +only needed for the repository hygiene checker): ```sh -python scripts/run_tests.py # all five Scriptorium suites -python scripts/run_tests.py --include-scribe # also test the pinned Scribe engine -python -m unittest discover -s tests/integration -v # HTTP port configuration -python -m unittest discover -s tests/tooling -v +wfl --execution-timeout 1200 scripts/run_tests.wfl # complete suite, including Scribe +wfl scripts/run_tests.wfl --group integration # focused HTTP workflows +wfl scripts/run_tests.wfl --group tooling # runner and hygiene regressions python scripts/check_repo_hygiene.py ``` Individual suites still run with `wfl --test TestPrograms/.test.wfl`. -The [WFL tests workflow](.github/workflows/wfl-tests.yml) runs all five -Scriptorium suites and the pinned Scribe suite on Blacksmith Linux using the +The [WFL tests workflow](.github/workflows/wfl-tests.yml) runs application, +ORM/migration/recovery, HTTP, tooling, examples and pinned Scribe suites on Blacksmith Linux using the latest `bsbyrdwfl/wfl:nightly` Docker image. It pulls the nightly tag on each run and records the resolved image digest, runtime version, and tested source revisions in the job summary. The Governance workflow checks tooling and @@ -248,7 +261,7 @@ tested against this Scriptorium — but it also means Scribe moving forward does ```sh scripts/update-scribe.sh --check # is there a newer Scribe? (changes nothing) scripts/update-scribe.sh # bump lib/scribe to the tip of Scribe main -python scripts/run_tests.py --include-scribe # app and upstream regression suites +wfl --execution-timeout 1200 scripts/run_tests.wfl # complete suite, including upstream Scribe git commit -m "chore(scribe): update lib/scribe" ``` diff --git a/REPOSITORY_HYGIENE.md b/REPOSITORY_HYGIENE.md index 68c171d..14a3ffc 100644 --- a/REPOSITORY_HYGIENE.md +++ b/REPOSITORY_HYGIENE.md @@ -22,7 +22,9 @@ deliberate. A migration needs its own proposal and maintainer decision. | Public themes | `themes/` | | Shipped CSS, fonts, logos, other source assets | `static/`, or the owning theme | | WFL behavior tests | `TestPrograms/` | -| Python automation regression tests | `tests/tooling/` | +| WFL automation regression tests and their WFL fixtures | `tests/tooling/` | +| WFL HTTP integration drivers and fixtures | `tests/integration/` | +| Executable ORM and migration examples | `examples/` (`*.test.wfl` is discovered) | | Contributor and CI automation | `scripts/`, `.github/` | | Maintained docs, designs, screenshots | `docs/` | | Shared agent instructions | [CLAUDE.md](CLAUDE.md); [AGENTS.md](AGENTS.md) is its discovery adapter | @@ -64,11 +66,12 @@ to keep checkout inspection portable and prevent reads outside the repository. ## Enforcement and its limits -Run from a checkout with Git and Python 3.11 or later: +Run from a checkout with WFL, Git and Python 3.11 or later. Python implements +the existing hygiene checker; its regression scenarios and fixtures are WFL: ```sh python scripts/check_repo_hygiene.py -python -m unittest discover -s tests/tooling -p "test_*.py" +wfl scripts/run_tests.wfl --group tooling ``` The [governance workflow](.github/workflows/governance.yml) runs the hygiene gate diff --git a/TestPrograms/.wflcfg b/TestPrograms/.wflcfg new file mode 100644 index 0000000..fe3befa --- /dev/null +++ b/TestPrograms/.wflcfg @@ -0,0 +1,6 @@ +# Suites include file-backed ORM, migration and recovery workflows. +# This bounds the whole suite; tests for locks/process/HTTP deadlines assert +# their own shorter limits. The WFL runner also owns a hard child timeout. +timeout_seconds = 180 +execution_logging = false +debug_report_enabled = false diff --git a/TestPrograms/db-contracts.test.wfl b/TestPrograms/db-contracts.test.wfl new file mode 100644 index 0000000..1b13a6c --- /dev/null +++ b/TestPrograms/db-contracts.test.wfl @@ -0,0 +1,98 @@ +// File-backed compatibility and failure contracts for the application adapter. +include from "../app/db.wfl" + +store artifact_dir as path_join of (path_dirname of script_directory) and "target" and "test-artifacts" and "db-contracts" +create directory at artifact_dir +store database_id as call generate_uuid +store database_path as path_join of artifact_dir and (database_id with ".db") +open database at ("sqlite://" with database_path) as conn +call db_migrate with conn + +define action called reset_contract_rows with parameters conn: + in transaction on conn: + store removed_trigger as execute conn with "DROP TRIGGER IF EXISTS reject_installer_completion" + for each table_name in ["sessions" and "users" and "posts" and "pages" and "settings" and "media" and "login_attempts"]: + store removed_rows as execute conn with ("DELETE FROM " with table_name) + end for + store removed_sequences as execute conn with "DELETE FROM sqlite_sequence" + end transaction +end action + +try: + describe "Application persistence compatibility": + test "empty settings and absent settings remain distinct": + call reset_contract_rows with conn + call setting_set with conn and "empty" and "" + call setting_default with conn and "empty" and "replacement" + expect setting_get of conn and "empty" and "fallback" to equal "" + expect setting_get of conn and "absent" and "fallback" to equal "fallback" + call setting_set with conn and "empty" and "updated" + expect setting_get of conn and "empty" and "fallback" to equal "updated" + end test + + test "content lookup and aliases preserve draft and orphan behavior": + call reset_contract_rows with conn + store inserted_post as post_create of conn and "draft" and "Draft" and "body" and "draft" and 777 + store post_row as post_by_slug of conn and "draft" + expect post_row["status"] to equal "draft" + expect post_row["author_id"] to equal 777 + expect post_row["author_name"] to equal nothing + store inserted_page as page_create of conn and "draft-page" and "Draft page" and "body" and "draft" and nothing + expect page_by_slug of conn and "draft-page" to equal nothing + store page_row as page_by_id of conn and inserted_page["last_insert_id"] + expect page_row["author_id"] to equal nothing + store inserted_media as media_create of conn and "orphan.bin" and "source.bin" and "application/octet-stream" and 3 and 777 + store media_rows as media_list_all of conn + store media_row as media_rows[0] + expect media_row["uploader_name"] to equal nothing + expect media_row["uploader_id"] to equal 777 + end test + + test "published ordering and page windows preserve exact projections": + call reset_contract_rows with conn + store first_post as post_create of conn and "first" and "First" and "one" and "published" and nothing + store second_post as post_create of conn and "second" and "Second" and "two" and "published" and nothing + store equal_times as execute conn with "UPDATE posts SET created_at = '2000-01-01 00:00:00'" + store first_page as post_list_published of conn and 1 and 0 + store second_page as post_list_published of conn and 1 and 1 + store first_row_value as first_page[0] + store second_row_value as second_page[0] + expect first_row_value["slug"] to equal "second" + expect second_row_value["slug"] to equal "first" + expect length of (post_list_published of conn and 0 and 0) to equal 0 + expect length of (post_list_published of conn and -1 and 0) to equal 2 + expect first_row_value["body_markdown"] to equal "two" + expect first_row_value["author_name"] to equal nothing + end test + + test "signed SQLite identities above floating point precision remain exact": + call reset_contract_rows with conn + store inserted_user as execute conn with "INSERT INTO users (id, username, password_hash, role) VALUES (9007199254740993, 'large', 'synthetic-hash', 'author')" + store found_user as user_by_id of conn and "9007199254740993" + expect found_user["id"] to equal "9007199254740993" + store next_user as user_create of conn and "next" and "synthetic-hash" and "author" + expect next_user["last_insert_id"] to equal "9007199254740994" + end test + + test "installer settings failure rolls back the user and earlier settings": + call reset_contract_rows with conn + call setting_set with conn and "site_title" and "Before" + store created_trigger as execute conn with "CREATE TRIGGER reject_installer_completion BEFORE INSERT ON settings WHEN NEW.skey = 'installed' BEGIN SELECT RAISE(ABORT, 'controlled installer failure'); END" + store rejected as no + try: + store installed_value as install_apply of conn and "After" and "Tagline" and "admin" and "synthetic-hash" + when error: + change rejected to yes + expect error_message contains "controlled installer failure" to be yes + end try + expect rejected to be yes + expect user_count of conn to equal 0 + expect setting_get of conn and "site_title" and "missing" to equal "Before" + expect setting_get of conn and "site_tagline" and "missing" to equal "missing" + expect install_is_done of conn to be no + end test + end describe +finally: + close database conn + delete file at database_path +end try diff --git a/TestPrograms/db-review-contracts.test.wfl b/TestPrograms/db-review-contracts.test.wfl new file mode 100644 index 0000000..205fb23 --- /dev/null +++ b/TestPrograms/db-review-contracts.test.wfl @@ -0,0 +1,138 @@ +// Independent review regressions: preserve legacy wrapper contracts and make +// ordinary SQLite nullable-key tables obey the same bulk-write page as reads. +include from "../app/db.wfl" + +describe "Independent persistence compatibility review": + test "null session equality does not delete a null primary-key row": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call user_create with conn and "reviewer" and "synthetic hash" and "author" + call session_create with conn and nothing and 1 and "2099-01-01 00:00:00" and "synthetic token" + store deleted as session_delete of conn and nothing + expect deleted["affected_rows"] to equal 0 + store remaining as query conn with "SELECT COUNT(*) AS total FROM sessions" + store remaining_row as remaining[0] + expect remaining_row["total"] to equal 1 + finally: + close database conn + end try + end test + + test "null author equality does not match orphan posts": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call post_create with conn and "orphan" and "Orphan" and "body" and "draft" and nothing + expect length of (post_list_by_author of conn and nothing) to equal 0 + finally: + close database conn + end try + end test + + test "null author equality does not match orphan pages": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call page_create with conn and "orphan" and "Orphan" and "body" and "draft" and nothing + expect length of (page_list_by_author of conn and nothing) to equal 0 + finally: + close database conn + end try + end test + + test "text primary-key insert retains native SQLite rowid metadata": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call user_create with conn and "reviewer" and "synthetic hash" and "author" + store inserted as session_create of conn and "review-session" and 1 and "2099-01-01 00:00:00" and "synthetic token" + expect inserted["affected_rows"] to equal 1 + expect inserted["last_insert_id"] to equal 1 + finally: + close database conn + end try + end test + + test "sensitive user update retains native connection insert metadata": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call user_create with conn and "reviewer" and "synthetic hash" and "author" + store updated as user_set_role of conn and 1 and "admin" + expect updated["affected_rows"] to equal 1 + expect updated["last_insert_id"] to equal 1 + finally: + close database conn + end try + end test + + test "expired null primary-key sessions are purged": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call user_create with conn and "reviewer" and "synthetic hash" and "author" + call session_create with conn and nothing and 1 and "2000-01-01 00:00:00" and "synthetic token" + store deleted as session_purge_expired of conn + expect deleted["affected_rows"] to equal 1 + store remaining as query conn with "SELECT COUNT(*) AS total FROM sessions" + store remaining_row as remaining[0] + expect remaining_row["total"] to equal 0 + finally: + close database conn + end try + end test + + // The following ORM contracts have no predecessor API in the baseline. + test "bulk update changes exactly the selected nullable-key row": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call setting_set with conn and nothing and "alpha" + call setting_set with conn and nothing and "bravo" + call setting_set with conn and "named" and "charlie" + store session as orm_borrow of conn + store requested as orm_whole_table of (orm_query of (db_settings_model)) + call orm_order_by with requested and "svalue" and "ascending" + call orm_page with requested and 1 and 1 + store selected as orm_find of session and requested + expect length of selected to equal 1 + expect orm_get of selected[0] and "svalue" to equal "bravo" + store updated as orm_update of session and requested and [(orm_value of "svalue" and "updated")] + expect updated["affected_rows"] to equal 1 + store actual_rows as query conn with "SELECT svalue FROM settings ORDER BY rowid" + store first_value as actual_rows[0] + store second_value as actual_rows[1] + store third_value as actual_rows[2] + expect first_value["svalue"] to equal "alpha" + expect second_value["svalue"] to equal "updated" + expect third_value["svalue"] to equal "charlie" + finally: + close database conn + end try + end test + + test "bulk delete removes exactly the selected nullable-key row": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call setting_set with conn and nothing and "alpha" + call setting_set with conn and nothing and "bravo" + call setting_set with conn and "named" and "charlie" + store session as orm_borrow of conn + store requested as orm_whole_table of (orm_query of (db_settings_model)) + call orm_order_by with requested and "svalue" and "ascending" + call orm_page with requested and 1 and 1 + store deleted as orm_delete of session and requested + expect deleted["affected_rows"] to equal 1 + store actual_rows as query conn with "SELECT svalue FROM settings ORDER BY rowid" + expect length of actual_rows to equal 2 + store first_value as actual_rows[0] + store second_value as actual_rows[1] + expect first_value["svalue"] to equal "alpha" + expect second_value["svalue"] to equal "charlie" + finally: + close database conn + end try + end test +end describe diff --git a/TestPrograms/migration-adoption.test.wfl b/TestPrograms/migration-adoption.test.wfl new file mode 100644 index 0000000..5609f5e --- /dev/null +++ b/TestPrograms/migration-adoption.test.wfl @@ -0,0 +1,35 @@ +include from "../app/db.wfl" + +store artifact_dir as path_join of (path_dirname of script_directory) and "target" and "test-artifacts" and "migration-adoption" +create directory at artifact_dir +store database_path as path_join of artifact_dir and "legacy.db" +check if file exists at database_path: + delete file at database_path +end check +open database at ("sqlite://" with database_path) as conn +try: + describe "Legacy migration adoption": + test "a supported sessions-only database gains audited history without losing its row": + in transaction on conn: + store created_table as execute conn with "CREATE TABLE sessions (id TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL)" + store seeded as execute conn with "INSERT INTO sessions (id,user_id,created_at,expires_at) VALUES ('legacy-session',9223372036854775806,NULL,'2099-01-01')" + end transaction + call db_migrate with conn + store saved as query conn with "SELECT id,CAST(user_id AS TEXT) AS user_id,created_at,csrf_token FROM sessions" + expect length of saved to equal 1 + store saved_row as saved[0] + expect saved_row["id"] to equal "legacy-session" + expect saved_row["user_id"] to equal "9223372036854775806" + expect saved_row["created_at"] to equal nothing + expect saved_row["csrf_token"] to equal "" + store history_tables as query conn with "SELECT count(*) AS n FROM sqlite_schema WHERE type='table' AND name='_orm_migrations'" + store history_table as history_tables[0] + expect history_table["n"] to equal 1 + end test + end describe +finally: + close database conn + check if file exists at database_path: + delete file at database_path + end check +end try diff --git a/TestPrograms/migration-engine.test.wfl b/TestPrograms/migration-engine.test.wfl new file mode 100644 index 0000000..db8d978 --- /dev/null +++ b/TestPrograms/migration-engine.test.wfl @@ -0,0 +1,99 @@ +include from "../tests/fixtures/migration-registry.wfl" + +describe "File-backed migration history and rollback": + test "plan is read only and targeted apply rollback count and reapply retain audit history": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store planned as orm_migration_plan of session and registry and "20260920000002" + expect length of planned.pending to equal 2 + expect orm_schema_exists of session and "_orm_migrations" to be no + store applied_plan as orm_migrate of session and registry and "20260920000002" + expect length of applied_plan.applied to equal 2 + store repeated_plan as orm_migrate of session and registry and "20260920000002" + expect length of repeated_plan.applied to equal 2 + store rolled_plan as orm_rollback_count of session and registry and 1 + expect length of rolled_plan.applied to equal 1 + store reapplied_plan as orm_migrate of session and registry and "20260920000002" + expect length of reapplied_plan.applied to equal 2 + store event_rows as orm_sql_query of session and "SELECT event_kind FROM _orm_migration_events ORDER BY event_id" and [] + expect length of event_rows to equal 4 + store rollback_event as event_rows[2] + expect rollback_event["event_kind"] to equal "rollback" + // Preflight must refuse the whole two-version rollback before the + // otherwise-reversible index migration has been rolled back. + store refused as no + try: + store impossible as orm_rollback_to of session and registry and "zero" + when error: + change refused to yes + expect error_message contains "irreversible" to be yes + end try + expect refused to be yes + store after_refusal as orm_migration_status of session and registry + expect length of after_refusal.applied to equal 2 + finally: + call migration_fixture_remove with session and database_path + end try + end test + + test "checksum edits missing files and missing ledger rows fail before changing managed data": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store applied_plan as orm_migrate of session and registry and "20260920000002" + store seeded as orm_sql_execute of session and "INSERT INTO items(title) VALUES (?)" and ["retained"] + store rejected as 0 + try: + store changed_registry as migration_fixture_registry of "fixture index source changed\n" + store changed_plan as orm_migration_status of session and changed_registry + when error: + change rejected to rejected plus 1 + expect error_message contains "checksum" to be yes + end try + try: + store short_plan as orm_migration_status of session and [registry[0]] + when error: + change rejected to rejected plus 1 + expect error_message contains "missing" to be yes + end try + store removed_history as orm_sql_execute of session and "DELETE FROM _orm_migrations WHERE position=2" and [] + try: + store corrupt_plan as orm_migration_status of session and registry + when error: + change rejected to rejected plus 1 + expect error_message contains "disagree" to be yes + end try + expect rejected to equal 3 + store kept_rows as orm_sql_query of session and "SELECT title FROM items" and [] + store kept_row as kept_rows[0] + expect kept_row["title"] to equal "retained" + finally: + call migration_fixture_remove with session and database_path + end try + end test + + test "checksum canonicalizes only CRLF and rejects drift even when history is unchanged": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry_lf as migration_fixture_registry of "first line\nsecond line\n" + store registry_crlf as migration_fixture_registry of "first line\r\nsecond line\r\n" + expect orm_migration_checksum of registry_lf[1] to equal (orm_migration_checksum of registry_crlf[1]) + store applied_plan as orm_migrate of session and registry_lf and "20260920000002" + store drifted as orm_sql_execute of session and "DROP INDEX idx_items_title" and [] + store refused as no + try: + store checked_plan as orm_migration_status of session and registry_lf + when error: + change refused to yes + expect error_message contains "missing managed index" to be yes + end try + expect refused to be yes + finally: + call migration_fixture_remove with session and database_path + end try + end test +end describe diff --git a/TestPrograms/migration-failures.test.wfl b/TestPrograms/migration-failures.test.wfl new file mode 100644 index 0000000..7f1af32 --- /dev/null +++ b/TestPrograms/migration-failures.test.wfl @@ -0,0 +1,99 @@ +include from "../tests/fixtures/migration-registry.wfl" + +describe "Migration failure boundaries": + test "a read-only SQLite URI fails with its cause and leaves history unchanged": + store database_path as migration_fixture_path + store owner_session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store initialized as orm_migrate of owner_session and registry and "20260920000002" + // SQLite's native file URI enforces read-only access on every + // pooled connection, even for a privileged CI operating-system user. + store uri_path as replace "\\" with "/" in database_path + store readonly_session as orm_open of ("file:" with uri_path with "?mode=ro") + try: + store refused as no + try: + store blocked_plan as orm_migrate of readonly_session and registry and "latest" + when error: + change refused to yes + expect error_message contains "readonly" to be yes + end try + expect refused to be yes + finally: + call orm_close with readonly_session + end try + store kept_plan as orm_migration_status of owner_session and registry + expect length of kept_plan.applied to equal 2 + finally: + call migration_fixture_remove with owner_session and database_path + end try + end test + + test "registry order duplicates empty scaffolds and destructive fake reversals fail before history exists": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store rejected as 0 + for each bad_registry in [[registry[0], registry[0]], [registry[1], registry[0]]]: + try: + store invalid_plan as orm_migration_plan of session and bad_registry and "latest" + when error: + change rejected to rejected plus 1 + end try + end for + create new OrmMigration as empty_version: + migration_id is "20260920000004" + migration_name is "Unfinished scaffold" + end + try: + store invalid_plan as orm_migration_plan of session and [empty_version] and "latest" + when error: + change rejected to rejected plus 1 + expect error_message contains "scaffold" to be yes + end try + store before_table as migration_fixture_table of yes and no + create new OrmMigration as destructive_version: + migration_id is "20260920000005" + migration_name is "Fake reverse" + managed_before is [before_table] + up_steps is [(orm_migration_drop_table of before_table)] + down_steps is [(orm_migration_create_table of before_table)] + end + try: + store invalid_plan as orm_migration_plan of session and [destructive_version] and "latest" + when error: + change rejected to rejected plus 1 + expect error_message contains "irreversible reason" to be yes + end try + expect rejected to equal 4 + expect orm_schema_exists of session and "_orm_migrations" to be no + finally: + call migration_fixture_remove with session and database_path + end try + end test + + test "a database path whose parent is a file reports a real open failure": + store owned_path as migration_fixture_path + open file at owned_path for writing as blocker_file + try: + wait for write content "synthetic path blocker" into blocker_file + finally: + close file blocker_file + end try + try: + store refused as no + try: + store invalid_session as orm_open of (path_join of owned_path and "site.db") + call orm_close with invalid_session + when error: + change refused to yes + end try + expect refused to be yes + expect file exists at owned_path to be yes + finally: + delete file at owned_path + end try + end test +end describe diff --git a/TestPrograms/migration-index-safety.test.wfl b/TestPrograms/migration-index-safety.test.wfl new file mode 100644 index 0000000..869be47 --- /dev/null +++ b/TestPrograms/migration-index-safety.test.wfl @@ -0,0 +1,39 @@ +include from "../tests/fixtures/migration-registry.wfl" + +describe "Historical managed index drift": + test "an index belonging to a later version cannot be silently accepted after rollback": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store initialized as orm_migrate of session and registry and "20260920000002" + store rolled_back as orm_rollback_to of session and registry and "20260920000001" + store reintroduced as orm_sql_execute of session and "CREATE INDEX idx_items_title ON items(title)" and [] + store refused as no + try: + store drifted_status as orm_migration_status of session and registry + when error: + change refused to yes + expect error_message contains "schema drift" to be yes + end try + expect refused to be yes + // SQLite object names are ASCII-case insensitive. A different + // spelling must not evade the historical managed-name check. + store conn as session.connection + in transaction on conn for schema changes: + store dropped_index as orm_sql_execute of session and "DROP INDEX idx_items_title" and [] + store upper_index as orm_sql_execute of session and "CREATE INDEX IDX_ITEMS_TITLE ON items(title)" and [] + end transaction + store upper_refused as no + try: + store upper_status as orm_migration_status of session and registry + when error: + change upper_refused to yes + expect error_message contains "schema drift" to be yes + end try + expect upper_refused to be yes + finally: + call migration_fixture_remove with session and database_path + end try + end test +end describe diff --git a/TestPrograms/migration-legacy-matrix.test.wfl b/TestPrograms/migration-legacy-matrix.test.wfl new file mode 100644 index 0000000..1074f23 --- /dev/null +++ b/TestPrograms/migration-legacy-matrix.test.wfl @@ -0,0 +1,118 @@ +include from "../tests/fixtures/legacy-database.wfl" + +describe "Inventoried legacy adoption states": + test "both full historical versions preserve all tables and extension objects": + for each with_csrf in [no, yes]: + store database_path as legacy_fixture_path + open database at ("sqlite://" with database_path) as conn + try: + in transaction on conn: + for each sql_text in (legacy_fixture_statements of with_csrf): + store created_table as execute conn with sql_text + end for + store user_row as execute conn with "INSERT INTO users(id,username,password_hash,role,created_at) VALUES(9007199254740993,'legacy-user','synthetic-existing-hash','extension-role',NULL)" + store session_row as execute conn with "INSERT INTO sessions(id,user_id,created_at,expires_at) VALUES('legacy-session',9007199254740993,NULL,'2099-01-01')" + check if with_csrf: + store token_row as execute conn with "UPDATE sessions SET csrf_token='synthetic-existing-token'" + end check + for each table_name in ["posts", "pages"]: + store content_row as execute conn with ("INSERT INTO " with table_name with "(id,slug,title,body_markdown,status,author_id,created_at,updated_at) VALUES(77,'kept','Kept title','','extension-status',999,NULL,'2001-01-01')") + end for + store settings_row as execute conn with "INSERT INTO settings(skey,svalue) VALUES('empty','')" + store media_row as execute conn with "INSERT INTO media(id,filename,original_name,content_type,size,uploader_id,created_at) VALUES(33,'kept.png','kept original','image/png',3,NULL,NULL)" + store attempt_row as execute conn with "INSERT INTO login_attempts(id,ip,attempted_at) VALUES(8,'192.0.2.10',NULL)" + store extension_table as execute conn with "CREATE TABLE extension_state(value TEXT)" + store extension_row as execute conn with "INSERT INTO extension_state VALUES('kept extension')" + store extension_index as execute conn with "CREATE INDEX extension_post_status ON posts(status)" + store extension_trigger as execute conn with "CREATE TRIGGER extension_post_write AFTER INSERT ON posts BEGIN INSERT INTO extension_state VALUES(new.title); END" + end transaction + call scriptorium_migrate with conn + call scriptorium_migrate with conn + store user_rows as query conn with "SELECT CAST(id AS TEXT) AS id,password_hash,role,created_at FROM users" + store saved_user as user_rows[0] + expect saved_user["id"] to equal "9007199254740993" + expect saved_user["password_hash"] to equal "synthetic-existing-hash" + expect saved_user["role"] to equal "extension-role" + expect saved_user["created_at"] to equal nothing + store sessions_rows as query conn with "SELECT csrf_token FROM sessions" + store saved_session as sessions_rows[0] + store expected_token as "" + check if with_csrf: + change expected_token to "synthetic-existing-token" + end check + expect saved_session["csrf_token"] to equal expected_token + for each table_name in ["posts", "pages"]: + store content_rows as query conn with ("SELECT status,author_id,created_at,updated_at FROM " with table_name) + store saved_content as content_rows[0] + expect saved_content["status"] to equal "extension-status" + expect saved_content["author_id"] to equal 999 + expect saved_content["created_at"] to equal nothing + expect saved_content["updated_at"] to equal "2001-01-01" + end for + store remaining_rows as query conn with "SELECT (SELECT count(*) FROM media WHERE uploader_id IS NULL AND created_at IS NULL) AS media_n,(SELECT count(*) FROM login_attempts WHERE attempted_at IS NULL) AS attempts_n,(SELECT count(*) FROM settings WHERE skey='empty' AND svalue='') AS settings_n" + store remaining as remaining_rows[0] + expect remaining["media_n"] to equal 1 + expect remaining["attempts_n"] to equal 1 + expect remaining["settings_n"] to equal 1 + store extension_rows as query conn with "SELECT name FROM sqlite_schema WHERE name IN ('extension_state','extension_post_status','extension_post_write')" + expect length of extension_rows to equal 3 + store history_rows as query conn with "SELECT event_kind FROM _orm_migration_events ORDER BY event_id" + expect length of history_rows to equal 2 + store initial_event as history_rows[0] + expect initial_event["event_kind"] to equal "adopt" + store second_event as history_rows[1] + store expected_event as "apply" + check if with_csrf: + change expected_event to "adopt" + end check + expect second_event["event_kind"] to equal expected_event + finally: + call legacy_fixture_remove with conn and database_path + end try + end for + end test + + test "sessions-only current legacy preserves its existing token": + store database_path as legacy_fixture_path + open database at ("sqlite://" with database_path) as conn + try: + in transaction on conn: + store sql_statements as legacy_fixture_statements of yes + store created_table as execute conn with sql_statements[1] + store session_row as execute conn with "INSERT INTO sessions(id,user_id,expires_at,csrf_token) VALUES('kept',7,'2099-01-01','existing token')" + end transaction + call scriptorium_migrate with conn + store kept_rows as query conn with "SELECT csrf_token FROM sessions WHERE id='kept'" + store kept_row as kept_rows[0] + expect kept_row["csrf_token"] to equal "existing token" + store history_rows as query conn with "SELECT * FROM _orm_migrations" + expect length of history_rows to equal 2 + finally: + call legacy_fixture_remove with conn and database_path + end try + end test + + test "unknown partial tables and changed constraints are never blindly stamped": + for each bad_sql in ["CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'author', created_at TEXT DEFAULT (datetime('now')))", "CREATE TABLE sessions (id TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL CHECK(length(expires_at)>0))"]: + store database_path as legacy_fixture_path + open database at ("sqlite://" with database_path) as conn + try: + store bad_table as execute conn with bad_sql + store refused as no + try: + call scriptorium_migrate with conn + when error: + change refused to yes + expect error_message contains "legacy adoption" to be yes + end try + expect refused to be yes + store history_rows as query conn with "SELECT name FROM sqlite_schema WHERE name='_orm_migrations'" + expect length of history_rows to equal 0 + store table_rows as query conn with "SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'" + expect length of table_rows to equal 1 + finally: + call legacy_fixture_remove with conn and database_path + end try + end for + end test +end describe diff --git a/TestPrograms/migration-rebuild.test.wfl b/TestPrograms/migration-rebuild.test.wfl new file mode 100644 index 0000000..774ce0c --- /dev/null +++ b/TestPrograms/migration-rebuild.test.wfl @@ -0,0 +1,92 @@ +include from "../tests/fixtures/migration-registry.wfl" + +describe "SQLite schema rebuild preservation": + test "an empty table retains its deleted AUTOINCREMENT high water through rebuild": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store initialized as orm_migrate of session and registry and "20260920000002" + store conn as session.connection + in transaction on conn: + store high_row as orm_sql_execute of session and "INSERT INTO items(id,title) VALUES(9007199254740998,'removed')" and [] + store removed_row as orm_sql_execute of session and "DELETE FROM items" and [] + end transaction + store rebuilt as orm_migrate of session and registry and "latest" + store inserted_row as orm_sql_execute of session and "INSERT INTO items(title) VALUES('after empty rebuild')" and [] + store saved_rows as orm_sql_query of session and "SELECT CAST(id AS TEXT) AS id FROM items" and [] + store saved_row as saved_rows[0] + expect saved_row["id"] to equal "9007199254740999" + finally: + call migration_fixture_remove with session and database_path + end try + end test + + test "rebuild preserves children extensions exact identities and deleted sequence high water": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store applied_plan as orm_migrate of session and registry and "20260920000002" + store conn as session.connection + in transaction on conn: + store child_table as orm_sql_execute of session and "CREATE TABLE extension_children(id INTEGER PRIMARY KEY,parent_id INTEGER REFERENCES items(id) ON DELETE CASCADE)" and [] + store audit_table as orm_sql_execute of session and "CREATE TABLE extension_audit(title TEXT)" and [] + store audit_trigger as orm_sql_execute of session and "CREATE TRIGGER extension_item_insert AFTER INSERT ON items BEGIN INSERT INTO extension_audit(title) VALUES (new.title); END" and [] + store extra_index as orm_sql_execute of session and "CREATE INDEX extension_title_index ON items(title DESC)" and [] + store parent_row as orm_sql_execute of session and "INSERT INTO items(id,title) VALUES (CAST(? AS INTEGER),?)" and ["9007199254740993", "retained"] + store child_row as orm_sql_execute of session and "INSERT INTO extension_children(id,parent_id) VALUES(1,CAST(? AS INTEGER))" and ["9007199254740993"] + store high_row as orm_sql_execute of session and "INSERT INTO items(id,title) VALUES(CAST(? AS INTEGER),?)" and ["9007199254740998", "deleted"] + store deleted_row as orm_sql_execute of session and "DELETE FROM items WHERE id=CAST(? AS INTEGER)" and ["9007199254740998"] + end transaction + store rebuilt as orm_migrate of session and registry and "latest" + expect length of rebuilt.applied to equal 3 + store kept_children as orm_sql_query of session and "SELECT CAST(parent_id AS TEXT) AS parent_id FROM extension_children" and [] + expect length of kept_children to equal 1 + store kept_child as kept_children[0] + expect kept_child["parent_id"] to equal "9007199254740993" + store generated_row as orm_sql_execute of session and "INSERT INTO items(title) VALUES ('next')" and [] + store generated_ids as orm_sql_query of session and "SELECT CAST(id AS TEXT) AS id FROM items WHERE title='next'" and [] + store generated_id as generated_ids[0] + expect generated_id["id"] to equal "9007199254740999" + store audit_rows as orm_sql_query of session and "SELECT count(*) AS n FROM extension_audit WHERE title='next'" and [] + store audit_row as audit_rows[0] + expect audit_row["n"] to equal 1 + store index_rows as orm_sql_query of session and "SELECT name FROM sqlite_schema WHERE name='extension_title_index'" and [] + expect length of index_rows to equal 1 + expect length of (orm_sql_query of session and "PRAGMA foreign_key_check" and []) to equal 0 + store reversed as orm_rollback_to of session and registry and "20260920000002" + expect length of reversed.applied to equal 2 + expect length of (orm_sql_query of session and "SELECT * FROM extension_children" and []) to equal 1 + finally: + call migration_fixture_remove with session and database_path + end try + end test + + test "a rejected rebuild rolls back schema data and history together and can recover": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store applied_plan as orm_migrate of session and registry and "latest" + store null_row as orm_sql_execute of session and "INSERT INTO items(title) VALUES(NULL)" and [] + store refused as no + try: + store impossible as orm_rollback_count of session and registry and 1 + when error: + change refused to yes + expect error_message contains "NOT NULL" to be yes + end try + expect refused to be yes + store checked_plan as orm_migration_status of session and registry + expect length of checked_plan.applied to equal 3 + expect length of (orm_sql_query of session and "SELECT id FROM items WHERE title IS NULL" and []) to equal 1 + expect orm_schema_exists of session and "_orm_rebuild_items" to be no + store repaired as orm_sql_execute of session and "UPDATE items SET title='repaired' WHERE title IS NULL" and [] + store recovered as orm_rollback_count of session and registry and 1 + expect length of recovered.applied to equal 2 + finally: + call migration_fixture_remove with session and database_path + end try + end test +end describe diff --git a/TestPrograms/migration-schema-safety.test.wfl b/TestPrograms/migration-schema-safety.test.wfl new file mode 100644 index 0000000..5caf7b2 --- /dev/null +++ b/TestPrograms/migration-schema-safety.test.wfl @@ -0,0 +1,80 @@ +include from "../tests/fixtures/migration-registry.wfl" + +describe "Independent schema preservation review": + test "quoted column whitespace is real schema drift": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store table_spec as migration_fixture_table of yes and no + store created as orm_sql_execute of session and "CREATE TABLE items (id INTEGER PRIMARY KEY AUTOINCREMENT, \"ti tle\" TEXT)" and [] + store problem_text as orm_schema_problem of session and table_spec + expect (length of problem_text) to be greater than 0 + finally: + call migration_fixture_remove with session and database_path + end try + end test + + test "a retained nullable column cannot silently disappear from rebuild copy": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store table_spec as migration_fixture_table of yes and no + create new OrmMigration as first_version: + migration_id is "20260920000001" + migration_name is "Create fixture" + managed_after is [table_spec] + up_steps is [(orm_migration_create_table of table_spec)] + irreversible_reason is "initial records" + end + create new OrmMigration as bad_version: + migration_id is "20260920000002" + migration_name is "Omitted retained column" + managed_before is [table_spec] + managed_after is [table_spec] + up_steps is [(orm_migration_rebuild of table_spec and table_spec and ["id"] and ["id"])] + down_steps is [(orm_migration_rebuild of table_spec and table_spec and ["id"] and ["id"])] + end + store initialized as orm_migrate of session and [first_version] and "latest" + store inserted as orm_sql_execute of session and "INSERT INTO items(title) VALUES ('must retain')" and [] + store refused as no + try: + store migrated as orm_migrate of session and [first_version, bad_version] and "latest" + when error: + change refused to yes + end try + store actual_rows as orm_sql_query of session and "SELECT title FROM items" and [] + store actual_row as actual_rows[0] + display "retained value after attempted migration: " with actual_row["title"] + expect refused to equal yes + expect actual_row["title"] to equal "must retain" + finally: + call migration_fixture_remove with session and database_path + end try + end test + + test "before-only managed objects are checked after a declared drop": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store table_spec as migration_fixture_table of yes and no + store created as orm_schema_create of session and table_spec + create new OrmMigration as removed_version: + migration_id is "20260920000001" + migration_name is "Drop pre-existing managed items" + managed_before is [table_spec] + managed_after is [] + up_steps is [(orm_migration_drop_table of table_spec)] + irreversible_reason is "drops records" + end + store refused as no + try: + store checked as orm_migration_schema_check of session and [removed_version] and 1 + when error: + change refused to yes + end try + expect refused to equal yes + finally: + call migration_fixture_remove with session and database_path + end try + end test +end describe diff --git a/TestPrograms/orm-crud.test.wfl b/TestPrograms/orm-crud.test.wfl new file mode 100644 index 0000000..96bf8e9 --- /dev/null +++ b/TestPrograms/orm-crud.test.wfl @@ -0,0 +1,218 @@ +include from "../tests/fixtures/orm-model.wfl" + +store artifact_dir as path_join of (path_dirname of script_directory) and "target" and "test-artifacts" and "orm-crud" +create directory at artifact_dir +store database_path as path_join of artifact_dir and "crud.db" +store session as orm_open of database_path +store entries as fixture_entries_model +try: + describe "File-backed ORM operations": + test "insert and read return typed records with defaults and exact identities": + call fixture_reset_entries with session + store saved as fixture_entry of session and entries and "first" + expect orm_get of saved and "id" to equal "1" + expect orm_get of saved and "enabled" to equal no + expect orm_get of saved and "score" to equal 7 + expect orm_get of saved and "note" to equal nothing + store found_record as orm_find_key of session and entries and "1" + expect orm_get of found_record and "slug" to equal "first" + expect orm_find_key of session and entries and "999" to equal nothing + expect saved.persisted to be yes + end test + + test "save changes assigned values while preserving the primary key": + call fixture_reset_entries with session + store saved as fixture_entry of session and entries and "before" + call orm_set with saved and "slug" and "after" + call orm_set with saved and "enabled" and yes + store updated as orm_save of session and saved + expect orm_get of updated and "slug" to equal "after" + expect orm_get of updated and "enabled" to equal yes + expect orm_get of updated and "id" to equal "1" + call orm_set with updated and "id" and "2" + store rejected as no + try: + store invalid_save as orm_save of session and updated + when error: + change rejected to yes + expect error_message contains "changing a persisted primary key" to be yes + end try + expect rejected to be yes + expect orm_count of session and (orm_query of entries) to equal 1 + end test + + test "bounded pages have stable ordering and counts describe the filter": + call fixture_reset_entries with session + store a as fixture_entry of session and entries and "bravo" + store b as fixture_entry of session and entries and "alpha" + store c as fixture_entry of session and entries and "charlie" + store requested as orm_query of entries + call orm_order_by with requested and "slug" and "ascending" + call orm_page with requested and 1 and 1 + store page_rows as orm_find of session and requested + expect length of page_rows to equal 1 + expect orm_get of page_rows[0] and "slug" to equal "bravo" + expect orm_count of session and requested to equal 3 + expect orm_exists of session and requested to be yes + store rejected as 0 + try: + call orm_page with requested and 0 and 0 + when error: + change rejected to rejected plus 1 + end try + try: + call orm_order_by with requested and "slug" and "ASC; DROP TABLE entries" + when error: + change rejected to rejected plus 1 + end try + try: + call orm_page with requested and 10 and -1 + when error: + change rejected to rejected plus 1 + end try + expect rejected to equal 3 + end test + + test "a projected omission stays missing and cannot save without its key": + call fixture_reset_entries with session + store saved as fixture_entry of session and entries and "projected" + store requested as orm_query of entries + call orm_select with requested and ["slug"] + store projected as orm_first of session and requested + expect orm_has of projected and "slug" to be yes + expect orm_has of projected and "note" to be no + store rejected as no + try: + store invalid_save as orm_save of session and projected + when error: + change rejected to yes + expect error_message contains "projected primary key" to be yes + end try + expect rejected to be yes + end test + + test "database uniqueness remains authoritative and upsert is explicit": + call fixture_reset_entries with session + store saved as fixture_entry of session and entries and "unique" + store duplicate_draft as orm_record of entries + call orm_set with duplicate_draft and "slug" and "unique" + call orm_set with duplicate_draft and "note" and "revised" + store rejected as no + try: + store duplicate as orm_save of session and duplicate_draft + when error: + change rejected to yes + expect error_message contains "UNIQUE constraint failed" to be yes + end try + expect rejected to be yes + expect orm_insert_if_absent of session and duplicate_draft and "slug" to equal nothing + store upserted as orm_upsert of session and duplicate_draft and "slug" + expect orm_get of upserted and "id" to equal "1" + expect orm_get of upserted and "note" to equal "revised" + expect orm_count of session and (orm_query of entries) to equal 1 + end test + + test "bulk writes require intent and honor page selection": + call fixture_reset_entries with session + store a as fixture_entry of session and entries and "a" + store b as fixture_entry of session and entries and "b" + store c as fixture_entry of session and entries and "c" + store requested as orm_query of entries + store rejected as no + try: + store unintended as orm_delete of session and requested + when error: + change rejected to yes + expect error_message contains "explicit orm_whole_table" to be yes + end try + expect rejected to be yes + call orm_where with requested and (orm_compare of "slug" and "in" and ["a" and "b"]) + call orm_page with requested and 1 and 0 + store changed_rows as orm_update of session and requested and [(orm_value of "note" and "changed")] + expect changed_rows["affected_rows"] to equal 1 + store b_record as orm_find_key of session and entries and "2" + expect orm_get of b_record and "note" to equal nothing + store deleted_rows as orm_delete of session and requested + expect deleted_rows["affected_rows"] to equal 1 + expect orm_count of session and (orm_query of entries) to equal 2 + store whole as orm_whole_table of (orm_query of entries) + store deleted_remaining as orm_delete of session and whole + expect deleted_remaining["affected_rows"] to equal 2 + end test + + test "validation failures propagate through native transaction rollback": + call fixture_reset_entries with session + store conn as session.connection + store rejected as no + try: + in transaction on conn: + store pending as fixture_entry of session and entries and "pending" + call orm_set with pending and "score" and "wrong type" + end transaction + when error: + change rejected to yes + expect error_message contains "whole number" to be yes + end try + expect rejected to be yes + expect orm_count of session and (orm_query of entries) to equal 0 + end test + + test "nested transactions reject and propagate rollback without corrupting the connection": + call fixture_reset_entries with session + store conn as session.connection + store rejected as no + try: + in transaction on conn: + store pending as fixture_entry of session and entries and "outer" + in transaction on conn: + store nested as fixture_entry of session and entries and "nested" + end transaction + end transaction + when error: + change rejected to yes + expect (to_lowercase of error_message) contains "transaction" to be yes + end try + expect rejected to be yes + expect orm_count of session and (orm_query of entries) to equal 0 + store after_failure as fixture_entry of session and entries and "after" + expect orm_count of session and (orm_query of entries) to equal 1 + end test + + test "large integer identities round trip without floating point": + call fixture_reset_entries with session + call orm_sql_execute with session and "INSERT INTO entries (id, slug) VALUES (9007199254740993, ?)" and ["large"] + store saved as orm_find_key of session and entries and "9007199254740993" + expect orm_get of saved and "id" to equal "9007199254740993" + call orm_set with saved and "note" and "exact" + store updated as orm_save of session and saved + expect orm_get of updated and "id" to equal "9007199254740993" + store next_record as fixture_entry of session and entries and "next" + expect orm_get of next_record and "id" to equal "9007199254740994" + end test + + test "borrowed connection ownership and inspectable query costs are explicit": + call fixture_reset_entries with session + store conn as session.connection + store borrowed as orm_borrow of conn + store secret_payload as "synthetic-secret-value" + store returned as orm_sql_query of borrowed and "SELECT ? AS label" and [secret_payload] + expect borrowed.query_count to equal 1 + expect borrowed.last_parameter_count to equal 1 + expect borrowed.last_sql contains secret_payload to be no + call orm_close with borrowed + call orm_close with borrowed + store rejected as no + try: + store invalid_read as orm_count of borrowed and (orm_query of entries) + when error: + change rejected to yes + expect error_message contains "session is closed" to be yes + end try + expect rejected to be yes + expect orm_count of session and (orm_query of entries) to equal 0 + end test + end describe +finally: + call orm_close with session + delete file at database_path +end try diff --git a/TestPrograms/orm-models.test.wfl b/TestPrograms/orm-models.test.wfl new file mode 100644 index 0000000..30814f6 --- /dev/null +++ b/TestPrograms/orm-models.test.wfl @@ -0,0 +1,97 @@ +include from "../lib/orm/models.wfl" + +create new OrmField as entry_key: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes +end +create new OrmField as entry_title: + field_name is "title" + value_type is "text" +end + +describe "Declarative ORM models": + test "fields, primary keys and indexes are discoverable": + create new OrmIndex as title_index: + index_name is "entries_title" + field_names is ["title"] + end + create new OrmModel as entries: + table_name is "entries" + fields is [entry_key and entry_title] + indexes is [title_index] + end + expect orm_model_problem of entries to equal "" + store found_field as orm_field_named of entries and "title" + expect found_field.value_type to equal "text" + expect orm_field_named of entries and "missing" to equal nothing + store key_spec as orm_primary_field of entries + expect key_spec.field_name to equal "id" + end test + + test "SQLite identifier case collisions are rejected": + create new OrmField as other_title: + field_name is "TITLE" + end + create new OrmModel as conflicting: + table_name is "entries" + fields is [entry_key and entry_title and other_title] + end + expect orm_model_problem of conflicting to equal "field names must be distinct without regard to ASCII case" + end test + + test "an index cannot quietly refer to a misspelled field": + create new OrmIndex as title_index: + index_name is "entries_title" + field_names is ["titel"] + end + create new OrmModel as entries: + table_name is "entries" + fields is [entry_key and entry_title] + indexes is [title_index] + end + expect orm_model_problem of entries to equal "an index refers to an undeclared field" + end test + + test "collection relationships retain a typed explicit definition": + create new OrmModel as entries: + table_name is "entries" + fields is [entry_key and entry_title] + end + create new OrmField as comment_owner: + field_name is "entry_id" + value_type is "identity" + end + create new OrmModel as comments: + table_name is "comments" + fields is [entry_key and comment_owner] + end + create new OrmRelationship as comments_relation: + relation_name is "comments" + related_model is comments + local_field is "id" + related_field is "entry_id" + many is yes + end + entries.relate(comments_relation) + expect orm_model_problem of entries to equal "" + expect length of entries.relationships to equal 1 + expect length of comments.relationships to equal 0 + end test + + test "one-record relationships require database uniqueness": + create new OrmModel as entries: + table_name is "entries" + fields is [entry_key and entry_title] + end + create new OrmRelationship as bad_relation: + relation_name is "same_title" + related_model is entries + local_field is "title" + related_field is "title" + end + entries.relate(bad_relation) + expect orm_model_problem of entries to equal "a single-record relationship must target a primary or unique field" + end test +end describe diff --git a/TestPrograms/orm-predicates.test.wfl b/TestPrograms/orm-predicates.test.wfl new file mode 100644 index 0000000..d2b7e84 --- /dev/null +++ b/TestPrograms/orm-predicates.test.wfl @@ -0,0 +1,127 @@ +include from "../lib/orm/predicates.wfl" + +create new OrmField as key_spec: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes +end +create new OrmField as title_spec: + field_name is "title" +end +create new OrmField as note_spec: + field_name is "note" + nullable is yes +end +create new OrmModel as entries: + table_name is "entries" + fields is [key_spec and title_spec and note_spec] +end + +define action called matching_ids with parameters conn and predicate: + store compiled as orm_compile_predicate of entries and predicate + return query conn with ("SELECT CAST(id AS TEXT) AS id FROM entries WHERE " with compiled.sql_text with " ORDER BY id") and parameters compiled.bound_values +end action + +store artifact_dir as path_join of (path_dirname of script_directory) and "target" and "test-artifacts" and "orm-predicates" +create directory at artifact_dir +store database_path as path_join of artifact_dir and "predicates.db" +open database at ("sqlite://" with database_path) as conn +try: + store cleared as execute conn with "DROP TABLE IF EXISTS entries" + store created as execute conn with "CREATE TABLE entries (id INTEGER PRIMARY KEY, title TEXT NOT NULL, note TEXT)" + store first_entry as execute conn with "INSERT INTO entries VALUES (1, '100%_done', NULL)" + store second_entry as execute conn with "INSERT INTO entries VALUES (2, '100XYZdone', '')" + store third_entry as execute conn with "INSERT INTO entries VALUES (3, 'third', 'memo')" + store fourth_entry as execute conn with "INSERT INTO entries VALUES (9007199254740993, ?, 'memo')" and parameters ["x'; DROP TABLE entries; --"] + + describe "SQL filters bind values and define null behavior": + test "malicious values remain bound and large identities stay exact": + store payload as "x'; DROP TABLE entries; --" + store predicate as orm_compare of "title" and "equal" and payload + store compiled as orm_compile_predicate of entries and predicate + expect compiled.sql_text contains payload to be no + expect compiled.bound_values[0] to equal payload + store matched_rows as matching_ids of conn and predicate + expect length of matched_rows to equal 1 + store match_row as matched_rows[0] + expect match_row["id"] to equal "9007199254740993" + store exact as matching_ids of conn and (orm_compare of "id" and "equal" and "9007199254740993") + expect length of exact to equal 1 + end test + + test "equality to null, inequality to null, and SQL negation are explicit": + store nulls as matching_ids of conn and (orm_compare of "note" and "equal" and nothing) + expect length of nulls to equal 1 + store not_nulls as matching_ids of conn and (orm_compare of "note" and "not equal" and nothing) + expect length of not_nulls to equal 3 + store not_memo as matching_ids of conn and (orm_not of (orm_compare of "note" and "equal" and "memo")) + expect length of not_memo to equal 1 + store empty_row as not_memo[0] + expect empty_row["id"] to equal "2" + end test + + test "all and any groups preserve parentheses and parameter ordering": + store selected_notes as orm_any of [(orm_compare of "note" and "equal" and nothing) and (orm_compare of "note" and "equal" and "memo")] + store selected as orm_all of [selected_notes and (orm_compare of "id" and "less than" and "4")] + store matched_rows as matching_ids of conn and selected + expect length of matched_rows to equal 2 + store first_match as matched_rows[0] + store second_match as matched_rows[1] + expect first_match["id"] to equal "1" + expect second_match["id"] to equal "3" + end test + + test "membership handles empty sets and explicit null membership": + store empty_matches as matching_ids of conn and (orm_compare of "id" and "in" and []) + expect length of empty_matches to equal 0 + store matched_rows as matching_ids of conn and (orm_compare of "note" and "in" and [nothing and "memo"]) + expect length of matched_rows to equal 3 + end test + + test "text searches treat wildcard characters as literal text": + store matched_rows as matching_ids of conn and (orm_compare of "title" and "contains" and "%_") + expect length of matched_rows to equal 1 + store found_entry as matched_rows[0] + expect found_entry["id"] to equal "1" + store starts as matching_ids of conn and (orm_compare of "title" and "starts with" and "100%") + expect length of starts to equal 1 + end test + + test "known tautologies remain visible to the bulk-write guard": + store unrestricted as orm_any of [(orm_compare of "title" and "equal" and "third") and (orm_all of [])] + store compiled as orm_compile_predicate of entries and unrestricted + expect compiled.constant_kind to equal "true" + store no_rows as orm_compile_predicate of entries and (orm_any of []) + expect no_rows.constant_kind to equal "false" + end test + + test "unknown identifiers, operators and null ordering fail safely": + store failures as 0 + try: + store malicious as orm_compile_predicate of entries and (orm_compare of "title; DROP TABLE entries" and "equal" and "x") + when error: + change failures to failures plus 1 + expect error_message contains "unknown field" to be yes + end try + try: + store malicious as orm_compile_predicate of entries and (orm_compare of "title" and "= ? OR 1=1" and "x") + when error: + change failures to failures plus 1 + expect error_message contains "unsupported comparison" to be yes + end try + try: + store invalid_null as orm_compile_predicate of entries and (orm_compare of "note" and "less than" and nothing) + when error: + change failures to failures plus 1 + expect error_message contains "null supports only" to be yes + end try + expect failures to equal 3 + store untouched as query conn with "SELECT id FROM entries" + expect length of untouched to equal 4 + end test + end describe +finally: + close database conn + delete file at database_path +end try diff --git a/TestPrograms/orm-query-order.test.wfl b/TestPrograms/orm-query-order.test.wfl new file mode 100644 index 0000000..29a856a --- /dev/null +++ b/TestPrograms/orm-query-order.test.wfl @@ -0,0 +1,23 @@ +include from "../tests/fixtures/orm-model.wfl" + +store artifact_dir as path_join of (path_dirname of script_directory) and "target" and "test-artifacts" and "orm-query-order" +create directory at artifact_dir +store database_path as path_join of artifact_dir and "ordering.db" +store session as orm_open of database_path +store entries as fixture_entries_model +try: + describe "Exact identity projection preserves SQLite ordering": + test "default ordering uses the integer column before projecting exact text": + call fixture_reset_entries with session + call orm_sql_execute with session and "INSERT INTO entries (id, slug) VALUES (1, 'first'), (2, 'second'), (10, 'tenth')" and [] + store records as orm_find of session and (orm_query of entries) + expect length of records to equal 3 + expect orm_get of records[0] and "id" to equal "1" + expect orm_get of records[1] and "id" to equal "2" + expect orm_get of records[2] and "id" to equal "10" + end test + end describe +finally: + call orm_close with session + delete file at database_path +end try diff --git a/TestPrograms/orm-records.test.wfl b/TestPrograms/orm-records.test.wfl new file mode 100644 index 0000000..4c56c27 --- /dev/null +++ b/TestPrograms/orm-records.test.wfl @@ -0,0 +1,106 @@ +include from "../lib/orm/records.wfl" + +create new OrmField as key_spec: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes +end +create new OrmField as name_spec: + field_name is "name" +end +create new OrmField as note_spec: + field_name is "note" + nullable is yes +end +create new OrmField as role_spec: + field_name is "role" + has_default is yes + default_value is "author" +end +create new OrmField as secret_spec: + field_name is "password_hash" + sensitive is yes +end +create new OrmModel as accounts: + table_name is "accounts" + fields is [key_spec and name_spec and note_spec and role_spec and secret_spec] +end + +describe "ORM records and safe assignment": + test "missing, null and empty text have distinct behavior": + store account as orm_record of accounts + expect orm_has of account and "note" to be no + store problem_text as "" + try: + store absent as orm_get of account and "note" + when error: + change problem_text to error_message + end try + expect problem_text contains "field is missing" to be yes + call orm_set with account and "note" and nothing + expect orm_has of account and "note" to be yes + expect orm_get of account and "note" to equal nothing + call orm_set with account and "note" and "" + expect orm_get of account and "note" to equal "" + expect orm_has of account and "role" to be no + end test + + test "unknown and sensitive batch fields reject the entire batch": + store account as orm_record of accounts + store problem_text as "" + try: + call orm_assign with account and [(orm_value of "name" and "Ada") and (orm_value of "password_hash" and "confidential")] + when error: + change problem_text to error_message + end try + expect problem_text contains "sensitive fields" to be yes + expect problem_text contains "confidential" to be no + expect orm_has of account and "name" to be no + change problem_text to "" + try: + call orm_set with account and "admin_typo" and yes + when error: + change problem_text to error_message + end try + expect problem_text contains "unknown field" to be yes + end test + + test "explicit sensitive assignment and defaults preserve declared intent": + store account as orm_record of accounts + call orm_set with account and "name" and "Ada" + call orm_set_sensitive with account and "password_hash" and "synthetic-hash" + expect orm_get of account and "password_hash" to equal "synthetic-hash" + expect orm_record_problem of account and yes to equal "" + expect orm_has of account and "role" to be no + expect orm_has of account and "id" to be no + end test + + test "record assignments are isolated and do not silently coerce types": + store first_account as orm_record of accounts + store second_account as orm_record of accounts + call orm_set with first_account and "name" and "Ada" + expect orm_has of second_account and "name" to be no + store problem_text as "" + try: + call orm_set with first_account and "name" and 123 + when error: + change problem_text to error_message + end try + expect problem_text contains "expected text" to be yes + expect orm_get of first_account and "name" to equal "Ada" + end test + + test "required omissions and duplicate assignments are actionable": + store account as orm_record of accounts + expect orm_record_problem of account and yes to equal "missing required field name; supply it before saving" + store problem_text as "" + try: + call orm_assign with account and [(orm_value of "name" and "Ada") and (orm_value of "name" and "Grace")] + when error: + change problem_text to error_message + end try + expect problem_text contains "cannot repeat a field" to be yes + expect orm_has of account and "name" to be no + end test +end describe diff --git a/TestPrograms/orm-relationship-limits.test.wfl b/TestPrograms/orm-relationship-limits.test.wfl new file mode 100644 index 0000000..ef2afbf --- /dev/null +++ b/TestPrograms/orm-relationship-limits.test.wfl @@ -0,0 +1,114 @@ +include from "../lib/orm/relationships.wfl" + +create new OrmField as key_spec: + field_name is "id" + value_type is "identity" + primary_key is yes +end +create new OrmField as name_spec: + field_name is "name" +end +create new OrmField as owner_spec: + field_name is "author_id" + value_type is "identity" + nullable is yes +end +create new OrmModel as authors: + table_name is "authors" + fields is [key_spec and name_spec] +end +create new OrmModel as notes: + table_name is "notes" + fields is [key_spec and owner_spec] +end +create new OrmRelationship as author_notes: + relation_name is "notes" + related_model is notes + local_field is "id" + related_field is "author_id" + many is yes +end +authors.relate(author_notes) +create new OrmRelationship as note_author: + relation_name is "author" + related_model is authors + local_field is "author_id" + related_field is "id" +end +notes.relate(note_author) + +store artifact_dir as path_join of (path_dirname of script_directory) and "target" and "test-artifacts" and "orm-relationship-limits" +create directory at artifact_dir +store database_path as path_join of artifact_dir and "limits.db" +store fixture_session as orm_open of database_path +store conn as fixture_session.connection +try: + in transaction on conn: + call orm_sql_execute with fixture_session and "DROP TABLE IF EXISTS notes" and [] + call orm_sql_execute with fixture_session and "DROP TABLE IF EXISTS authors" and [] + call orm_sql_execute with fixture_session and "CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT NOT NULL)" and [] + call orm_sql_execute with fixture_session and "CREATE TABLE notes (id INTEGER PRIMARY KEY, author_id INTEGER)" and [] + call orm_sql_execute with fixture_session and "INSERT INTO authors VALUES (1, 'one')" and [] + call orm_sql_execute with fixture_session and "WITH RECURSIVE n(id) AS (SELECT 1 UNION ALL SELECT id + 1 FROM n WHERE id < 10001) INSERT INTO notes SELECT id, 1 FROM n" and [] + end transaction + describe "Relationship bounds": + test "more than one query page is fully loaded with a bounded query count": + store session as orm_borrow of conn + // Fixture reduction occurs transactionally, preserving the next case. + in transaction on conn: + call orm_sql_execute with session and "UPDATE notes SET author_id = 2 WHERE id > 1001" and [] + store author_record as orm_find_key of session and authors and "1" + store before_queries as session.query_count + call orm_load with session and [author_record] and "notes" + expect session.query_count minus before_queries to equal 2 + store loaded_notes as orm_related of author_record and "notes" + expect length of loaded_notes to equal 1001 + expect orm_get of loaded_notes[1000] and "id" to equal "1001" + call orm_sql_execute with session and "UPDATE notes SET author_id = 1" and [] + end transaction + end test + test "exceeding the eager limit fails without exposing a partial relationship": + store session as orm_borrow of conn + store author_record as orm_find_key of session and authors and "1" + store before_queries as session.query_count + store rejected as no + try: + call orm_load with session and [author_record] and "notes" + when error: + change rejected to yes + expect error_message contains "limited to 10000" to be yes + end try + expect rejected to be yes + expect session.query_count minus before_queries to equal 11 + change rejected to no + try: + store incomplete as orm_related of author_record and "notes" + when error: + change rejected to yes + expect error_message contains "not loaded" to be yes + end try + expect rejected to be yes + end test + test "oversized parent batches fail before issuing a query": + store session as orm_borrow of conn + store author_record as orm_find_key of session and authors and "1" + store oversized as [] + count from 1 to 1001: + push with oversized and author_record + end count + store before_queries as session.query_count + store rejected as no + try: + call orm_load with session and oversized and "notes" + when error: + change rejected to yes + expect error_message contains "at most 1000" to be yes + end try + expect rejected to be yes + expect session.query_count to equal before_queries + end test + end describe +finally: + call orm_close with fixture_session + delete file at database_path +end try diff --git a/TestPrograms/orm-relationships.test.wfl b/TestPrograms/orm-relationships.test.wfl new file mode 100644 index 0000000..face9ce --- /dev/null +++ b/TestPrograms/orm-relationships.test.wfl @@ -0,0 +1,121 @@ +include from "../lib/orm/relationships.wfl" + +create new OrmField as key_spec: + field_name is "id" + value_type is "identity" + primary_key is yes +end +create new OrmField as name_spec: + field_name is "name" +end +create new OrmField as owner_spec: + field_name is "author_id" + value_type is "identity" + nullable is yes +end +create new OrmModel as authors: + table_name is "authors" + fields is [key_spec and name_spec] +end +create new OrmModel as notes: + table_name is "notes" + fields is [key_spec and owner_spec] +end +create new OrmRelationship as author_notes: + relation_name is "notes" + related_model is notes + local_field is "id" + related_field is "author_id" + many is yes +end +authors.relate(author_notes) +create new OrmRelationship as note_author: + relation_name is "author" + related_model is authors + local_field is "author_id" + related_field is "id" +end +notes.relate(note_author) + +store artifact_dir as path_join of (path_dirname of script_directory) and "target" and "test-artifacts" and "orm-relationships" +create directory at artifact_dir +store database_path as path_join of artifact_dir and "relationships.db" +store fixture_session as orm_open of database_path +store conn as fixture_session.connection +try: + call orm_sql_execute with fixture_session and "DROP TABLE IF EXISTS notes" and [] + call orm_sql_execute with fixture_session and "DROP TABLE IF EXISTS authors" and [] + call orm_sql_execute with fixture_session and "CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT NOT NULL)" and [] + call orm_sql_execute with fixture_session and "CREATE TABLE notes (id INTEGER PRIMARY KEY, author_id INTEGER)" and [] + call orm_sql_execute with fixture_session and "WITH RECURSIVE n(id) AS (SELECT 1 UNION ALL SELECT id + 1 FROM n WHERE id < 101) INSERT INTO authors SELECT id, 'author-' || id FROM n" and [] + call orm_sql_execute with fixture_session and "INSERT INTO notes VALUES (1, 1), (2, 1), (3, 101), (4, 999), (5, NULL)" and [] + + describe "Explicit and batched relationships": + test "collection loading uses two queries for 101 distinct parent keys": + store session as orm_borrow of conn + store requested as orm_query of authors + call orm_page with requested and 1000 and 0 + store author_records as orm_find of session and requested + expect length of author_records to equal 101 + store before_queries as session.query_count + call orm_load with session and author_records and "notes" + expect session.query_count minus before_queries to equal 2 + store first_notes as orm_related of author_records[0] and "notes" + store last_notes as orm_related of author_records[100] and "notes" + store empty_notes as orm_related of author_records[50] and "notes" + expect length of first_notes to equal 2 + expect length of last_notes to equal 1 + expect length of empty_notes to equal 0 + expect orm_get of first_notes[0] and "id" to equal "1" + expect orm_get of first_notes[1] and "id" to equal "2" + expect session.query_count minus before_queries to equal 2 + end test + + test "single-record loads preserve orphan and null behavior with no lazy SQL": + store session as orm_borrow of conn + store note_records as orm_find of session and (orm_query of notes) + store before_queries as session.query_count + store rejected as no + try: + store unexpected as orm_related of note_records[0] and "author" + when error: + change rejected to yes + expect error_message contains "relationship is not loaded" to be yes + end try + expect rejected to be yes + expect session.query_count to equal before_queries + call orm_load with session and note_records and "author" + expect session.query_count minus before_queries to equal 1 + store first_author as orm_related of note_records[0] and "author" + expect orm_get of first_author and "name" to equal "author-1" + expect orm_related of note_records[3] and "author" to equal nothing + expect orm_related of note_records[4] and "author" to equal nothing + expect session.query_count minus before_queries to equal 1 + end test + + test "an empty batch costs no queries and missing projected keys fail before SQL": + store session as orm_borrow of conn + store before_queries as session.query_count + store no_records as orm_load of session and [] and "notes" + expect length of no_records to equal 0 + expect session.query_count to equal before_queries + store requested as orm_query of authors + call orm_select with requested and ["name"] + call orm_page with requested and 1 and 0 + store projected as orm_find of session and requested + change before_queries to session.query_count + store rejected as no + try: + call orm_load with session and projected and "notes" + when error: + change rejected to yes + expect error_message contains "field is missing" to be yes + end try + expect rejected to be yes + expect session.query_count to equal before_queries + end test + end describe +finally: + call orm_close with fixture_session + delete file at database_path +end try diff --git a/TestPrograms/orm-types.test.wfl b/TestPrograms/orm-types.test.wfl new file mode 100644 index 0000000..fa99361 --- /dev/null +++ b/TestPrograms/orm-types.test.wfl @@ -0,0 +1,95 @@ +include from "../lib/orm/types.wfl" + +describe "ORM declaration and value contracts": + test "identifiers accept ASCII names but reject SQL fragments": + expect orm_identifier_problem of "entries_2026" to equal "" + expect orm_identifier_problem of "_private" to equal "" + expect orm_identifier_problem of "a.b" to equal "use an ASCII letter or underscore followed by ASCII letters, digits, or underscores" + expect orm_identifier_problem of "name; DROP TABLE users" to equal "use an ASCII letter or underscore followed by ASCII letters, digits, or underscores" + expect orm_identifier_problem of "9name" to equal "use an ASCII letter or underscore followed by ASCII letters, digits, or underscores" + expect orm_identifier_problem of "" to equal "use an ASCII letter or underscore followed by ASCII letters, digits, or underscores" + end test + + test "integer identities are exact canonical text": + expect orm_identity_problem of "9007199254740993" to equal "" + expect orm_identity_problem of "9223372036854775807" to equal "" + expect orm_identity_problem of "-9223372036854775808" to equal "" + expect orm_identity_problem of "9223372036854775808" to equal "identity is outside SQLite's signed 64-bit range" + expect orm_identity_problem of "-9223372036854775809" to equal "identity is outside SQLite's signed 64-bit range" + expect orm_identity_problem of "01" to equal "use canonical integer text without spaces, a plus sign, or leading zeros" + expect orm_identity_problem of "1.5" to equal "use canonical integer text without spaces, a plus sign, or leading zeros" + expect orm_identity_problem of "-0" to equal "use canonical integer text without spaces, a plus sign, or leading zeros" + end test + + test "nullable text distinguishes null from empty text and other types": + create new OrmField as caption: + field_name is "caption" + value_type is "text" + nullable is yes + end + expect orm_value_problem of caption and nothing to equal "" + expect orm_value_problem of caption and "" to equal "" + expect orm_value_problem of caption and 7 to equal "expected text" + create new OrmField as title_field: + field_name is "title" + value_type is "text" + end + expect orm_value_problem of title_field and nothing to equal "null is not allowed; supply a value or omit the field to use its declared default" + end test + + test "integer arithmetic values reject fractions and imprecise numbers": + create new OrmField as quantity: + field_name is "quantity" + value_type is "integer" + end + expect orm_value_problem of quantity and 12 to equal "" + expect orm_value_problem of quantity and -12 to equal "" + expect orm_value_problem of quantity and 1.5 to equal "expected a whole number in the exactly representable range" + expect orm_value_problem of quantity and "12" to equal "expected a whole number in the exactly representable range" + expect orm_value_problem of quantity and 9007199254740992 to equal "expected a whole number in the exactly representable range" + end test + + test "boolean and number fields do not coerce strings": + create new OrmField as enabled_field: + field_name is "enabled" + value_type is "boolean" + end + create new OrmField as score_field: + field_name is "score" + value_type is "number" + end + expect orm_value_problem of enabled_field and yes to equal "" + expect orm_value_problem of enabled_field and 1 to equal "expected boolean" + expect orm_value_problem of enabled_field and "yes" to equal "expected boolean" + expect orm_value_problem of score_field and 1.25 to equal "" + expect orm_value_problem of score_field and "1.25" to equal "expected a finite number" + end test + + test "declarations check defaults, generated keys and supported types": + create new OrmField as key_field: + field_name is "entry_id" + value_type is "identity" + primary_key is yes + generated is yes + end + expect orm_field_problem of key_field to equal "" + create new OrmField as bad_key: + field_name is "entry_id" + value_type is "text" + generated is yes + end + expect orm_field_problem of bad_key to equal "generated fields must be non-null identity primary keys" + create new OrmField as bad_default: + field_name is "label" + value_type is "text" + has_default is yes + default_value is 3 + end + expect orm_field_problem of bad_default to equal "invalid default: expected text" + create new OrmField as unknown_type: + field_name is "payload" + value_type is "anything" + end + expect orm_field_problem of unknown_type to equal "supported field types are text, integer, number, boolean, and identity" + end test +end describe diff --git a/TestPrograms/orm-write-review.test.wfl b/TestPrograms/orm-write-review.test.wfl new file mode 100644 index 0000000..7fd1ca6 --- /dev/null +++ b/TestPrograms/orm-write-review.test.wfl @@ -0,0 +1,84 @@ +include from "../app/db.wfl" + +define action called collision_model: + create new OrmField as key_spec: + field_name is "id" + nullable is yes + primary_key is yes + end + create new OrmModel as model_spec: + table_name is "review_aliases" + fields is [key_spec and (db_text_field of "_ROWID_" and no) and (db_text_field of "rowid" and no) and (db_text_field of "_orm_insert_rowid" and no) and (db_text_field of "note" and no)] + end + return model_spec +end action + +describe "Independent row identity and sensitive write review": + test "declared aliases preserve real values and physical page selection": + open database at "sqlite::memory:" as conn + try: + store created as execute conn with "CREATE TABLE review_aliases (id TEXT PRIMARY KEY, _ROWID_ TEXT NOT NULL, rowid TEXT NOT NULL, _orm_insert_rowid TEXT NOT NULL, note TEXT NOT NULL)" + store session as orm_borrow of conn + store model_spec as call collision_model + count from 1 to 2: + store draft as orm_record of model_spec + call orm_assign with draft and [(orm_value of "id" and nothing) and (orm_value of "_ROWID_" and "logical first alias") and (orm_value of "rowid" and "logical second alias") and (orm_value of "_orm_insert_rowid" and "logical metadata field") and (orm_value of "note" and "tie")] + store saved as orm_insert of session and draft + expect saved.inserted_rowid to equal ("" with count) + expect orm_get of saved and "_ROWID_" to equal "logical first alias" + expect orm_get of saved and "rowid" to equal "logical second alias" + expect orm_get of saved and "_orm_insert_rowid" to equal "logical metadata field" + end count + store requested as orm_whole_table of (orm_query of model_spec) + call orm_page with requested and 1 and 1 + store updated as orm_update of session and requested and [(orm_value of "note" and "updated")] + expect updated["affected_rows"] to equal 1 + store actual_rows as query conn with "SELECT note FROM review_aliases ORDER BY oid" + store first_value as actual_rows[0] + store second_value as actual_rows[1] + expect first_value["note"] to equal "tie" + expect second_value["note"] to equal "updated" + finally: + close database conn + end try + end test + + test "sensitive bulk counterpart keeps assignment and query guards": + open database at "sqlite::memory:" as conn + try: + call db_migrate with conn + call user_create with conn and "reviewer" and "synthetic hash" and "author" + store session as orm_borrow of conn + store requested as orm_query of (db_users_model) + store rejected as 0 + try: + store bad_value as orm_update_sensitive of session and requested and "role" and "admin" + when error: + change rejected to rejected plus 1 + end try + call orm_where with requested and (orm_compare of "id" and "equal" and "1") + try: + store bad_value as orm_update_sensitive of session and requested and "id" and "2" + when error: + change rejected to rejected plus 1 + end try + try: + store bad_value as orm_update_sensitive of session and requested and "role" and 42 + when error: + change rejected to rejected plus 1 + end try + expect rejected to equal 3 + store user_row as user_by_id of conn and 1 + expect user_row["role"] to equal "author" + expect user_by_id of conn and 2 to equal nothing + store literal_value as "admin', password_hash='injected" + store changed_value as orm_update_sensitive of session and requested and "role" and literal_value + store after_row as user_by_id of conn and 1 + expect changed_value["affected_rows"] to equal 1 + expect after_row["role"] to equal literal_value + expect after_row["password_hash"] to equal "synthetic hash" + finally: + close database conn + end try + end test +end describe diff --git a/app/auth.wfl b/app/auth.wfl index 654f8c7..7fe4de2 100644 --- a/app/auth.wfl +++ b/app/auth.wfl @@ -32,9 +32,7 @@ end action // Expiry (7 days) is computed in SQL so we don't depend on WFL date formatting. define action called session_start with parameters conn and user_id: store sid as new_session_id - store exp_rows as query conn with "SELECT datetime('now', '+7 days') AS exp" - store exp_row as exp_rows[0] - store expires_at as exp_row["exp"] + store expires_at as db_clock of conn and "+7 days" store csrf as generate_csrf_token store created as session_create of conn and sid and user_id and expires_at and csrf return sid diff --git a/app/db.wfl b/app/db.wfl index 36e6e0e..d4985a6 100644 --- a/app/db.wfl +++ b/app/db.wfl @@ -1,47 +1,11 @@ -// Scriptorium — data layer. -// -// Owns the SQLite schema and every query/execute the app makes. Callers pass in -// the open connection handle (`conn`) so this file stays free of global state and -// is easy to unit-test against `sqlite::memory:`. -// -// Timestamps are filled by SQLite itself (DEFAULT (datetime('now')) and explicit -// datetime('now') in updates) so we never depend on WFL date formatting. -// -// Dependency chain: db -> util -include from "util.wfl" - -// --------------------------------------------------------------------------- -// Schema — idempotent. Safe to run on every boot (no migration engine in WFL). -// --------------------------------------------------------------------------- +// Compatibility API: models own row access; migrations own schema. +// The caller owns its native connection. +include from "persistence-helpers.wfl" + define action called db_migrate with parameters conn: - store r1 as execute conn with "CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'author', created_at TEXT DEFAULT (datetime('now')))" - store r2 as execute conn with "CREATE TABLE IF NOT EXISTS sessions (id TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL, csrf_token TEXT NOT NULL DEFAULT '')" - store r3 as execute conn with "CREATE TABLE IF NOT EXISTS posts (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT UNIQUE NOT NULL, title TEXT NOT NULL, body_markdown TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'draft', author_id INTEGER, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')))" - store r4 as execute conn with "CREATE TABLE IF NOT EXISTS pages (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT UNIQUE NOT NULL, title TEXT NOT NULL, body_markdown TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'draft', author_id INTEGER, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')))" - store r5 as execute conn with "CREATE TABLE IF NOT EXISTS settings (skey TEXT PRIMARY KEY, svalue TEXT NOT NULL DEFAULT '')" - store r6 as execute conn with "CREATE TABLE IF NOT EXISTS media (id INTEGER PRIMARY KEY AUTOINCREMENT, filename TEXT UNIQUE NOT NULL, original_name TEXT NOT NULL DEFAULT '', content_type TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0, uploader_id INTEGER, created_at TEXT DEFAULT (datetime('now')))" - store r7 as execute conn with "CREATE TABLE IF NOT EXISTS login_attempts (id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, attempted_at TEXT DEFAULT (datetime('now')))" - // The rate limiter filters on (ip, attempted_at) on every login POST — - // exactly when the table is growing fastest — so keep that path indexed. - store r7b as execute conn with "CREATE INDEX IF NOT EXISTS idx_login_attempts_ip_time ON login_attempts (ip, attempted_at)" - // Databases created before the CSRF work lack sessions.csrf_token. SQLite - // has no ADD COLUMN IF NOT EXISTS, so ask the schema whether the column is - // there and only ALTER when it is missing. - // - // This deliberately does NOT wrap the ALTER in a `try`. Catching every error - // would treat "column already exists" and "database is locked / read-only" - // identically, so migration would report success on a database it had failed - // to upgrade — and the failure would resurface later as session_create - // blowing up on a missing csrf_token column, far from the cause. - store csrf_cols as query conn with "SELECT count(*) AS n FROM pragma_table_info('sessions') WHERE name = 'csrf_token'" - store csrf_col_row as csrf_cols[0] - check if csrf_col_row["n"] is equal to 0: - store r8 as execute conn with "ALTER TABLE sessions ADD COLUMN csrf_token TEXT NOT NULL DEFAULT ''" - end check - return yes + return scriptorium_migrate of conn end action -// first_row: return row 0 of a result list, or nothing when empty. define action called first_row with parameters rows: check if (length of rows) is greater than 0: return rows[0] @@ -49,38 +13,33 @@ define action called first_row with parameters rows: return nothing end action -// --------------------------------------------------------------------------- -// Settings — a simple key/value store. -// --------------------------------------------------------------------------- define action called setting_get with parameters conn and setting_key and fallback: - store rows as query conn with "SELECT svalue FROM settings WHERE skey = ?" and parameters [setting_key] - store row0 as first_row of rows - check if row0 is nothing: + check if setting_key is nothing: return fallback end check - return row0["svalue"] + store record_value as orm_find_key of (orm_borrow of conn) and (db_settings_model) and setting_key + check if record_value is nothing: + return fallback + end check + return orm_get of record_value and "svalue" end action define action called setting_set with parameters conn and setting_key and setting_value: - store r as execute conn with "INSERT INTO settings (skey, svalue) VALUES (?, ?) ON CONFLICT(skey) DO UPDATE SET svalue = ?" and parameters [setting_key and setting_value and setting_value] + store record_value as orm_record of (db_settings_model) + call orm_assign with record_value and [(orm_value of "skey" and setting_key) and (orm_value of "svalue" and setting_value)] + store saved as orm_upsert of (orm_borrow of conn) and record_value and "skey" return yes end action -// Insert a default only if the key is absent (used at seed time). define action called setting_default with parameters conn and setting_key and setting_value: - store r as execute conn with "INSERT OR IGNORE INTO settings (skey, svalue) VALUES (?, ?)" and parameters [setting_key and setting_value] + store record_value as orm_record of (db_settings_model) + call orm_assign with record_value and [(orm_value of "skey" and setting_key) and (orm_value of "svalue" and setting_value)] + store saved as orm_insert_if_absent of (orm_borrow of conn) and record_value and "skey" return yes end action -// --------------------------------------------------------------------------- -// First-run installer lock (`settings.installed` = "yes"). -// --------------------------------------------------------------------------- define action called install_is_done with parameters conn: - store flag as setting_get of conn and "installed" and "" - check if flag is equal to "yes": - return yes - end check - return no + return (setting_get of conn and "installed" and "") is equal to "yes" end action define action called install_mark_done with parameters conn: @@ -88,233 +47,296 @@ define action called install_mark_done with parameters conn: return yes end action -// Create the first admin, write site title/tagline, then lock the wizard. -// Unique-username failure (and any other user_create error) returns no and -// leaves settings and the installed flag untouched. +// A rejected first user returns false as before. Later failures propagate only +// after the transaction has rolled back both the user and installation settings. define action called install_apply with parameters conn and site_title and site_tagline and username and password_hash: - store applied as yes + store creating_user as yes try: - call user_create with conn and username and password_hash and "admin" + in transaction on conn: + check if (install_is_done of conn) or (user_count of conn) is greater than 0: + call orm_fail with "installation" and "the site already has an administrator; use normal administration" + end check + call user_create with conn and username and password_hash and "admin" + change creating_user to no + call setting_set with conn and "site_title" and site_title + call setting_set with conn and "site_tagline" and site_tagline + call install_mark_done with conn + end transaction when error: - change applied to no + check if creating_user: + return no + end check + call raise_error with error_message end try - check if applied is equal to no: - return no - end check - call setting_set with conn and "site_title" and site_title - call setting_set with conn and "site_tagline" and site_tagline - call install_mark_done with conn return yes end action -// --------------------------------------------------------------------------- -// Users -// --------------------------------------------------------------------------- define action called user_count with parameters conn: - store rows as query conn with "SELECT count(*) AS n FROM users" - store row0 as rows[0] - return row0["n"] + return orm_count of (orm_borrow of conn) and (orm_query of (db_users_model)) end action define action called user_create with parameters conn and username and password_hash and role_name: - return execute conn with "INSERT INTO users (username, password_hash, role) VALUES (?, ?, ?)" and parameters [username and password_hash and role_name] + store record_value as orm_record of (db_users_model) + call orm_set with record_value and "username" and username + call orm_set_sensitive with record_value and "password_hash" and password_hash + call orm_set_sensitive with record_value and "role" and role_name + return db_insert_result of (orm_save of (orm_borrow of conn) and record_value) end action define action called user_by_username with parameters conn and username: - store rows as query conn with "SELECT id, username, password_hash, role, created_at FROM users WHERE username = ?" and parameters [username] - return first_row of rows + store requested as db_lookup of (db_users_model) and "username" and username + return db_user_map of (orm_first of (orm_borrow of conn) and requested) and yes end action define action called user_by_id with parameters conn and the_id: - store rows as query conn with "SELECT id, username, password_hash, role, created_at FROM users WHERE id = ?" and parameters [the_id] - return first_row of rows + return db_user_map of (orm_find_key of (orm_borrow of conn) and (db_users_model) and (db_identity of the_id)) and yes end action define action called user_list with parameters conn: - store rows as query conn with "SELECT id, username, role, created_at FROM users ORDER BY id" - return rows + store requested as orm_query of (db_users_model) + call orm_select with requested and ["id" and "username" and "role" and "created_at"] + call orm_order_by with requested and "id" and "ascending" + store records as db_read_records of (orm_borrow of conn) and requested and -1 and 0 and "" + store mapped_rows as [] + for each record_value in records: + push with mapped_rows and (db_user_map of record_value and no) + end for + return mapped_rows +end action + +define action called db_user_sensitive_update with parameters conn and the_id and field_name and field_value: + store session as orm_borrow of conn + store requested as db_lookup of (db_users_model) and "id" and (db_identity of the_id) + return orm_update_sensitive of session and requested and field_name and field_value end action define action called user_set_role with parameters conn and the_id and role_name: - return execute conn with "UPDATE users SET role = ? WHERE id = ?" and parameters [role_name and the_id] + return db_user_sensitive_update of conn and the_id and "role" and role_name end action define action called user_set_password with parameters conn and the_id and password_hash: - return execute conn with "UPDATE users SET password_hash = ? WHERE id = ?" and parameters [password_hash and the_id] + return db_user_sensitive_update of conn and the_id and "password_hash" and password_hash end action define action called user_delete with parameters conn and the_id: - return execute conn with "DELETE FROM users WHERE id = ?" and parameters [the_id] + return orm_delete of (orm_borrow of conn) and (db_lookup of (db_users_model) and "id" and (db_identity of the_id)) end action -// --------------------------------------------------------------------------- -// Sessions -// --------------------------------------------------------------------------- define action called session_create with parameters conn and session_id and user_id and expires_at and csrf_token: - return execute conn with "INSERT INTO sessions (id, user_id, expires_at, csrf_token) VALUES (?, ?, ?, ?)" and parameters [session_id and user_id and expires_at and csrf_token] + store record_value as orm_record of (db_sessions_model) + call orm_set_sensitive with record_value and "id" and session_id + call orm_set_sensitive with record_value and "csrf_token" and csrf_token + call orm_assign with record_value and [(orm_value of "user_id" and (db_identity of user_id)) and (orm_value of "expires_at" and expires_at)] + store saved as orm_save of (orm_borrow of conn) and record_value + return db_insert_result of saved end action -// Returns the joined user row for a live (non-expired) session, or nothing. -// Carries the session's csrf_token so handlers can validate POSTed forms. define action called session_user with parameters conn and session_id: - store rows as query conn with "SELECT u.id AS id, u.username AS username, u.role AS role, s.csrf_token AS csrf_token FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.id = ? AND s.expires_at > datetime('now')" and parameters [session_id] - return first_row of rows + check if session_id is nothing: + return nothing + end check + store session as orm_borrow of conn + store requested as orm_query of (db_sessions_model) + call orm_where with requested and (orm_all of [(orm_compare of "id" and "equal" and session_id) and (orm_compare of "expires_at" and "greater than" and (db_clock of conn and "+0 seconds"))]) + store session_record as orm_first of session and requested + check if session_record is nothing: + return nothing + end check + call orm_load with session and [session_record] and "user" + store user_record as orm_related of session_record and "user" + check if user_record is nothing: + return nothing + end check + create map result_value: + "id" is db_legacy_identity of (orm_get of user_record and "id") + "username" is orm_get of user_record and "username" + "role" is orm_get of user_record and "role" + "csrf_token" is orm_get of session_record and "csrf_token" + end map + return result_value end action define action called session_delete with parameters conn and session_id: - return execute conn with "DELETE FROM sessions WHERE id = ?" and parameters [session_id] + return db_delete_matching of (orm_borrow of conn) and (db_lookup of (db_sessions_model) and "id" and session_id) end action define action called session_purge_expired with parameters conn: - return execute conn with "DELETE FROM sessions WHERE expires_at <= datetime('now')" + store requested as orm_query of (db_sessions_model) + call orm_where with requested and (orm_compare of "expires_at" and "less or equal" and (db_clock of conn and "+0 seconds")) + return db_delete_matching of (orm_borrow of conn) and requested end action -// --------------------------------------------------------------------------- -// Login attempts — per-IP failure log backing the login rate limiter. -// Timestamps and the window are SQL-side (datetime('now')) like everything else. -// --------------------------------------------------------------------------- define action called login_attempt_record with parameters conn and ip: - return execute conn with "INSERT INTO login_attempts (ip) VALUES (?)" and parameters [ip] + store record_value as orm_record of (db_login_attempts_model) + call orm_set with record_value and "ip" and ip + return db_insert_result of (orm_save of (orm_borrow of conn) and record_value) end action -// Failed attempts from this IP in the last 15 minutes. define action called login_attempts_recent with parameters conn and ip: - store rows as query conn with "SELECT count(*) AS n FROM login_attempts WHERE ip = ? AND attempted_at > datetime('now', '-15 minutes')" and parameters [ip] - store row0 as rows[0] - return row0["n"] + store requested as orm_query of (db_login_attempts_model) + call orm_where with requested and (orm_all of [(orm_compare of "ip" and "equal" and ip) and (orm_compare of "attempted_at" and "greater than" and (db_clock of conn and "-15 minutes"))]) + return orm_count of (orm_borrow of conn) and requested end action -// A successful login wipes the counter for that IP. define action called login_attempts_clear with parameters conn and ip: - return execute conn with "DELETE FROM login_attempts WHERE ip = ?" and parameters [ip] + return db_delete_matching of (orm_borrow of conn) and (db_lookup of (db_login_attempts_model) and "ip" and ip) end action -// Housekeeping: drop attempts too old to ever count against the window. define action called login_attempts_purge with parameters conn: - return execute conn with "DELETE FROM login_attempts WHERE attempted_at <= datetime('now', '-15 minutes')" + store requested as orm_query of (db_login_attempts_model) + call orm_where with requested and (orm_compare of "attempted_at" and "less or equal" and (db_clock of conn and "-15 minutes")) + return db_delete_matching of (orm_borrow of conn) and requested end action -// --------------------------------------------------------------------------- -// Media — uploaded files; the bytes live under static/uploads/, rows here. -// --------------------------------------------------------------------------- define action called media_create with parameters conn and filename and original_name and content_type and size and uploader_id: - return execute conn with "INSERT INTO media (filename, original_name, content_type, size, uploader_id) VALUES (?, ?, ?, ?, ?)" and parameters [filename and original_name and content_type and size and uploader_id] + store record_value as orm_record of (db_media_model) + call orm_assign with record_value and [(orm_value of "filename" and filename) and (orm_value of "original_name" and original_name) and (orm_value of "content_type" and content_type) and (orm_value of "size" and size) and (orm_value of "uploader_id" and (db_identity of uploader_id))] + return db_insert_result of (orm_save of (orm_borrow of conn) and record_value) end action define action called media_list_all with parameters conn: - store rows as query conn with "SELECT m.id AS id, m.filename AS filename, m.original_name AS original_name, m.content_type AS content_type, m.size AS size, m.uploader_id AS uploader_id, m.created_at AS created_at, u.username AS uploader_name FROM media m LEFT JOIN users u ON u.id = m.uploader_id ORDER BY m.created_at DESC, m.id DESC" - return rows + store requested as orm_query of (db_media_model) + call orm_order_by with requested and "created_at" and "descending" + call orm_order_by with requested and "id" and "descending" + store records as db_read_records of (orm_borrow of conn) and requested and -1 and 0 and "uploader" + store mapped_rows as [] + for each record_value in records: + push with mapped_rows and (db_media_map of record_value and yes) + end for + return mapped_rows end action define action called media_by_id with parameters conn and the_id: - store rows as query conn with "SELECT id, filename, original_name, content_type, size, uploader_id, created_at FROM media WHERE id = ?" and parameters [the_id] - return first_row of rows + return db_media_map of (orm_find_key of (orm_borrow of conn) and (db_media_model) and (db_identity of the_id)) and no end action define action called media_delete with parameters conn and the_id: - return execute conn with "DELETE FROM media WHERE id = ?" and parameters [the_id] + return orm_delete of (orm_borrow of conn) and (db_lookup of (db_media_model) and "id" and (db_identity of the_id)) +end action + +define action called db_content_create with parameters conn and table_name and slug and title and body_markdown and the_status and author_id: + store record_value as orm_record of (db_content_model of table_name) + call orm_assign with record_value and [(orm_value of "slug" and slug) and (orm_value of "title" and title) and (orm_value of "body_markdown" and body_markdown) and (orm_value of "status" and the_status) and (orm_value of "author_id" and (db_identity of author_id))] + return db_insert_result of (orm_save of (orm_borrow of conn) and record_value) +end action + +define action called db_content_update with parameters conn and table_name and the_id and slug and title and body_markdown and the_status: + store requested as db_lookup of (db_content_model of table_name) and "id" and (db_identity of the_id) + store revised_values as [(orm_value of "slug" and slug) and (orm_value of "title" and title) and (orm_value of "body_markdown" and body_markdown) and (orm_value of "status" and the_status) and (orm_value of "updated_at" and (db_clock of conn and "+0 seconds"))] + return orm_update of (orm_borrow of conn) and requested and revised_values end action -// --------------------------------------------------------------------------- -// Posts -// --------------------------------------------------------------------------- define action called post_create with parameters conn and slug and title and body_markdown and the_status and author_id: - return execute conn with "INSERT INTO posts (slug, title, body_markdown, status, author_id) VALUES (?, ?, ?, ?, ?)" and parameters [slug and title and body_markdown and the_status and author_id] + return db_content_create of conn and "posts" and slug and title and body_markdown and the_status and author_id end action define action called post_update with parameters conn and the_id and slug and title and body_markdown and the_status: - return execute conn with "UPDATE posts SET slug = ?, title = ?, body_markdown = ?, status = ?, updated_at = datetime('now') WHERE id = ?" and parameters [slug and title and body_markdown and the_status and the_id] + return db_content_update of conn and "posts" and the_id and slug and title and body_markdown and the_status end action define action called post_delete with parameters conn and the_id: - return execute conn with "DELETE FROM posts WHERE id = ?" and parameters [the_id] + return orm_delete of (orm_borrow of conn) and (db_lookup of (db_content_model of "posts") and "id" and (db_identity of the_id)) +end action + +define action called db_post_find with parameters conn and field_name and field_value: + store session as orm_borrow of conn + store record_value as orm_first of session and (db_lookup of (db_content_model of "posts") and field_name and field_value) + check if record_value is not nothing: + call orm_load with session and [record_value] and "author" + end check + return db_post_map of record_value and "full" end action define action called post_by_id with parameters conn and the_id: - store rows as query conn with "SELECT p.id AS id, p.slug AS slug, p.title AS title, p.body_markdown AS body_markdown, p.status AS status, p.author_id AS author_id, p.created_at AS created_at, p.updated_at AS updated_at, u.username AS author_name FROM posts p LEFT JOIN users u ON u.id = p.author_id WHERE p.id = ?" and parameters [the_id] - return first_row of rows + return db_post_find of conn and "id" and (db_identity of the_id) end action define action called post_by_slug with parameters conn and slug: - store rows as query conn with "SELECT p.id AS id, p.slug AS slug, p.title AS title, p.body_markdown AS body_markdown, p.status AS status, p.author_id AS author_id, p.created_at AS created_at, p.updated_at AS updated_at, u.username AS author_name FROM posts p LEFT JOIN users u ON u.id = p.author_id WHERE p.slug = ?" and parameters [slug] - return first_row of rows + return db_post_find of conn and "slug" and slug +end action + +define action called db_post_list with parameters conn and requested and projection_kind and page_limit and page_offset: + store records as db_read_records of (orm_borrow of conn) and requested and page_limit and page_offset and "author" + store mapped_rows as [] + for each record_value in records: + push with mapped_rows and (db_post_map of record_value and projection_kind) + end for + return mapped_rows end action -// Published posts, newest first, one page at a time. define action called post_list_published with parameters conn and page_limit and page_offset: - store rows as query conn with "SELECT p.id AS id, p.slug AS slug, p.title AS title, p.body_markdown AS body_markdown, p.status AS status, p.created_at AS created_at, u.username AS author_name FROM posts p LEFT JOIN users u ON u.id = p.author_id WHERE p.status = 'published' ORDER BY p.created_at DESC, p.id DESC LIMIT ? OFFSET ?" and parameters [page_limit and page_offset] - return rows + store requested as db_lookup of (db_content_model of "posts") and "status" and "published" + call orm_order_by with requested and "created_at" and "descending" + call orm_order_by with requested and "id" and "descending" + return db_post_list of conn and requested and "published" and page_limit and page_offset end action define action called post_count_published with parameters conn: - store rows as query conn with "SELECT count(*) AS n FROM posts WHERE status = 'published'" - store row0 as rows[0] - return row0["n"] + return orm_count of (orm_borrow of conn) and (db_lookup of (db_content_model of "posts") and "status" and "published") end action -// All posts for the admin list (optionally scoped to one author). define action called post_list_all with parameters conn: - store rows as query conn with "SELECT p.id AS id, p.slug AS slug, p.title AS title, p.status AS status, p.author_id AS author_id, p.updated_at AS updated_at, u.username AS author_name FROM posts p LEFT JOIN users u ON u.id = p.author_id ORDER BY p.updated_at DESC, p.id DESC" - return rows + store requested as orm_query of (db_content_model of "posts") + call orm_order_by with requested and "updated_at" and "descending" + call orm_order_by with requested and "id" and "descending" + return db_post_list of conn and requested and "admin" and -1 and 0 end action define action called post_list_by_author with parameters conn and author_id: - store rows as query conn with "SELECT p.id AS id, p.slug AS slug, p.title AS title, p.status AS status, p.author_id AS author_id, p.updated_at AS updated_at, u.username AS author_name FROM posts p LEFT JOIN users u ON u.id = p.author_id WHERE p.author_id = ? ORDER BY p.updated_at DESC, p.id DESC" and parameters [author_id] - return rows + store requested as db_lookup of (db_content_model of "posts") and "author_id" and (db_identity of author_id) + call orm_order_by with requested and "updated_at" and "descending" + call orm_order_by with requested and "id" and "descending" + return db_post_list of conn and requested and "admin" and -1 and 0 end action -// --------------------------------------------------------------------------- -// Pages -// --------------------------------------------------------------------------- define action called page_create with parameters conn and slug and title and body_markdown and the_status and author_id: - return execute conn with "INSERT INTO pages (slug, title, body_markdown, status, author_id) VALUES (?, ?, ?, ?, ?)" and parameters [slug and title and body_markdown and the_status and author_id] + return db_content_create of conn and "pages" and slug and title and body_markdown and the_status and author_id end action define action called page_update with parameters conn and the_id and slug and title and body_markdown and the_status: - return execute conn with "UPDATE pages SET slug = ?, title = ?, body_markdown = ?, status = ?, updated_at = datetime('now') WHERE id = ?" and parameters [slug and title and body_markdown and the_status and the_id] + return db_content_update of conn and "pages" and the_id and slug and title and body_markdown and the_status end action define action called page_delete with parameters conn and the_id: - return execute conn with "DELETE FROM pages WHERE id = ?" and parameters [the_id] + return orm_delete of (orm_borrow of conn) and (db_lookup of (db_content_model of "pages") and "id" and (db_identity of the_id)) end action define action called page_by_id with parameters conn and the_id: - store rows as query conn with "SELECT id, slug, title, body_markdown, status, author_id, updated_at FROM pages WHERE id = ?" and parameters [the_id] - return first_row of rows + return db_page_map of (orm_find_key of (orm_borrow of conn) and (db_content_model of "pages") and (db_identity of the_id)) and "full" end action define action called page_by_slug with parameters conn and slug: - store rows as query conn with "SELECT id, slug, title, body_markdown, status, author_id, updated_at FROM pages WHERE slug = ? AND status = 'published'" and parameters [slug] - return first_row of rows + store requested as orm_query of (db_content_model of "pages") + call orm_where with requested and (orm_all of [(orm_compare of "slug" and "equal" and slug) and (orm_compare of "status" and "equal" and "published")]) + return db_page_map of (orm_first of (orm_borrow of conn) and requested) and "full" +end action + +define action called db_page_list with parameters conn and requested and projection_kind: + call orm_order_by with requested and "title" and "ascending" + store records as db_read_records of (orm_borrow of conn) and requested and -1 and 0 and "" + store mapped_rows as [] + for each record_value in records: + push with mapped_rows and (db_page_map of record_value and projection_kind) + end for + return mapped_rows end action define action called page_list_all with parameters conn: - store rows as query conn with "SELECT id, slug, title, status, updated_at FROM pages ORDER BY title" - return rows + return db_page_list of conn and (orm_query of (db_content_model of "pages")) and "admin" end action define action called page_list_by_author with parameters conn and author_id: - store rows as query conn with "SELECT id, slug, title, status, updated_at FROM pages WHERE author_id = ? ORDER BY title" and parameters [author_id] - return rows + return db_page_list of conn and (db_lookup of (db_content_model of "pages") and "author_id" and (db_identity of author_id)) and "admin" end action -// Published pages, for the site nav. define action called page_list_published with parameters conn: - store rows as query conn with "SELECT id, slug, title FROM pages WHERE status = 'published' ORDER BY title" - return rows + return db_page_list of conn and (db_lookup of (db_content_model of "pages") and "status" and "published") and "navigation" end action -// --------------------------------------------------------------------------- -// Counts (dashboard) -// --------------------------------------------------------------------------- define action called posts_total with parameters conn: - store rows as query conn with "SELECT count(*) AS n FROM posts" - store row0 as rows[0] - return row0["n"] + return orm_count of (orm_borrow of conn) and (orm_query of (db_content_model of "posts")) end action define action called pages_total with parameters conn: - store rows as query conn with "SELECT count(*) AS n FROM pages" - store row0 as rows[0] - return row0["n"] + return orm_count of (orm_borrow of conn) and (orm_query of (db_content_model of "pages")) end action diff --git a/app/migrations.wfl b/app/migrations.wfl new file mode 100644 index 0000000..4e03a67 --- /dev/null +++ b/app/migrations.wfl @@ -0,0 +1,96 @@ +include from "util.wfl" +include from "../lib/orm/migrations.wfl" +include from "migrations/20200102000000_session_csrf.wfl" + +define action called scriptorium_migration_source with parameters version_file: + store source_path as path_join of current_directory and "app/migrations" and version_file + check if (file exists at source_path) is equal to no: + call orm_fail with "migration source" and "an immutable migration file is missing; run from the application root and restore the complete version registry" + end check + open file at source_path for reading as source_file + try: + wait for store source_text as read content from source_file + return source_text + finally: + close file source_file + end try +end action + +define action called scriptorium_migration_registry: + return [(migration_20200101000000 of (scriptorium_migration_source of "20200101000000_initial.wfl")), (migration_20200102000000 of (scriptorium_migration_source of "20200102000000_session_csrf.wfl"))] +end action + +// Adoption supports exactly the inventoried full pre-CSRF/full current schema, +// or a sessions-only fixture/site at either version. Unknown partial tables and +// altered constraints fail before history or data changes. Unmanaged extension +// tables, indexes and triggers are retained and never entered in the ledger. +define action called scriptorium_legacy_version with parameters session and registry: + store first_version as registry[0] + store second_version as registry[1] + store present_names as [] + store csrf_present as no + for each table_spec in first_version.managed_after: + check if orm_schema_exists of session and table_spec.model_spec.table_name: + push with present_names and table_spec.model_spec.table_name + store expected_table as table_spec + check if table_spec.model_spec.table_name is equal to "sessions": + store columns_found as orm_sql_query of session and "SELECT name FROM pragma_table_info('sessions') WHERE name='csrf_token'" and [] + check if (length of columns_found) is equal to 1: + change csrf_present to yes + change expected_table to second_version.managed_after[1] + end check + end check + store schema_problem as orm_schema_problem of session and expected_table + check if schema_problem is not equal to "": + call orm_fail with "legacy adoption" and schema_problem + end check + end check + end for + check if (length of present_names) is equal to 0: + return 0 + end check + check if (length of present_names) is not equal to 7: + check if (length of present_names) is not equal to 1 or present_names[0] is not equal to "sessions": + call orm_fail with "legacy adoption" and "unsupported partial managed schema; supported states are empty, all seven legacy tables, or sessions only; restore a complete matching backup" + end check + end check + check if csrf_present: + return 2 + end check + return 1 +end action + +define action called scriptorium_adopt with parameters session and registry: + store conn as session.connection + in transaction on conn for schema changes: + // Both history presence and every supported legacy constraint are read + // again under the same bounded write lock used to record adoption. + store applied_rows as orm_migration_history of session and registry + check if (orm_schema_exists of session and "_orm_migrations") is equal to no: + store legacy_version as scriptorium_legacy_version of session and registry + check if legacy_version is greater than 0: + store final_version as registry[legacy_version minus 1] + for each table_spec in final_version.managed_after: + check if (orm_schema_exists of session and table_spec.model_spec.table_name) is equal to no: + store created_missing as orm_schema_create of session and table_spec + end check + end for + store verified_tables as orm_schema_require of session and final_version.managed_after + store history_ready as orm_migration_ensure_history of session + count from 1 to legacy_version: + store version_spec as registry[count minus 1] + store adopted_version as orm_migration_record of session and version_spec and count and "adopt" + end count + end check + end check + end transaction + return yes +end action + +define action called scriptorium_migrate with parameters conn: + store session as orm_borrow of conn + store registry as scriptorium_migration_registry + store adopted as scriptorium_adopt of session and registry + store migrated as orm_migrate of session and registry and "latest" + return yes +end action diff --git a/app/migrations/20200101000000_initial.wfl b/app/migrations/20200101000000_initial.wfl new file mode 100644 index 0000000..b5b1b92 --- /dev/null +++ b/app/migrations/20200101000000_initial.wfl @@ -0,0 +1,94 @@ +// Immutable historical schema. Never import app/models.wfl here. +define action called historical_field with parameters field_name and type_name and nullable_value and primary_value and generated_value and unique_value and has_default_value and default_value and server_value: + create new OrmField as field_spec: + field_name is field_name + value_type is type_name + nullable is nullable_value + primary_key is primary_value + generated is generated_value + unique_value is unique_value + has_default is has_default_value + default_value is default_value + server_default is server_value + end + return field_spec +end action + +define action called historical_identity: + return historical_field of "id" and "identity" and no and yes and yes and no and no and nothing and "" +end action + +define action called historical_text with parameters field_name and nullable_value: + return historical_field of field_name and "text" and nullable_value and no and no and no and no and nothing and "" +end action + +define action called historical_default with parameters field_name and default_value: + return historical_field of field_name and "text" and no and no and no and no and yes and default_value and "" +end action + +define action called historical_timestamp with parameters field_name: + return historical_field of field_name and "text" and yes and no and no and no and no and nothing and "current timestamp" +end action + +define action called historical_content_table with parameters table_name: + store field_specs as [historical_identity, (historical_field of "slug" and "text" and no and no and no and yes and no and nothing and ""), (historical_text of "title" and no), (historical_default of "body_markdown" and ""), (historical_default of "status" and "draft"), (historical_field of "author_id" and "identity" and yes and no and no and no and no and nothing and ""), (historical_timestamp of "created_at"), (historical_timestamp of "updated_at")] + create new OrmModel as model_spec: + table_name is table_name + fields is field_specs + end + return orm_schema_table of model_spec +end action + +define action called historical_initial_tables: + store historical_tables as [] + create new OrmModel as users_model: + table_name is "users" + fields is [historical_identity, (historical_field of "username" and "text" and no and no and no and yes and no and nothing and ""), (historical_text of "password_hash" and no), (historical_default of "role" and "author"), (historical_timestamp of "created_at")] + end + push with historical_tables and (orm_schema_table of users_model) + create new OrmModel as sessions_model: + table_name is "sessions" + fields is [(historical_field of "id" and "text" and yes and yes and no and no and no and nothing and ""), (historical_field of "user_id" and "identity" and no and no and no and no and no and nothing and ""), (historical_timestamp of "created_at"), (historical_text of "expires_at" and no)] + end + push with historical_tables and (orm_schema_table of sessions_model) + push with historical_tables and (historical_content_table of "posts") + push with historical_tables and (historical_content_table of "pages") + create new OrmModel as settings_model: + table_name is "settings" + fields is [(historical_field of "skey" and "text" and yes and yes and no and no and no and nothing and ""), (historical_default of "svalue" and "")] + end + push with historical_tables and (orm_schema_table of settings_model) + create new OrmModel as media_model: + table_name is "media" + fields is [historical_identity, (historical_field of "filename" and "text" and no and no and no and yes and no and nothing and ""), (historical_default of "original_name" and ""), (historical_default of "content_type" and ""), (historical_field of "size" and "integer" and no and no and no and no and yes and 0 and ""), (historical_field of "uploader_id" and "identity" and yes and no and no and no and no and nothing and ""), (historical_timestamp of "created_at")] + end + push with historical_tables and (orm_schema_table of media_model) + create new OrmIndex as attempts_index: + index_name is "idx_login_attempts_ip_time" + field_names is ["ip", "attempted_at"] + end + create new OrmModel as attempts_model: + table_name is "login_attempts" + fields is [historical_identity, (historical_text of "ip" and no), (historical_timestamp of "attempted_at")] + indexes is [attempts_index] + end + push with historical_tables and (orm_schema_table of attempts_model) + return historical_tables +end action + +define action called migration_20200101000000 with parameters immutable_source: + store historical_tables as historical_initial_tables + store create_steps as [] + for each table_spec in historical_tables: + push with create_steps and (orm_migration_create_table of table_spec) + end for + create new OrmMigration as migration_spec: + migration_id is "20200101000000" + migration_name is "Initial Scriptorium schema" + up_steps is create_steps + irreversible_reason is "removing the initial schema would destroy site records" + managed_after is historical_tables + source_text is immutable_source + end + return migration_spec +end action diff --git a/app/migrations/20200102000000_session_csrf.wfl b/app/migrations/20200102000000_session_csrf.wfl new file mode 100644 index 0000000..ed777bd --- /dev/null +++ b/app/migrations/20200102000000_session_csrf.wfl @@ -0,0 +1,39 @@ +include from "20200101000000_initial.wfl" + +define action called historical_csrf_tables: + store revised_tables as [] + for each table_spec in historical_initial_tables: + check if table_spec.model_spec.table_name is equal to "sessions": + store revised_fields as [] + for each field_spec in table_spec.model_spec.fields: + push with revised_fields and field_spec + end for + push with revised_fields and (historical_default of "csrf_token" and "") + create new OrmModel as revised_model: + table_name is "sessions" + fields is revised_fields + end + push with revised_tables and (orm_schema_table of revised_model) + otherwise: + push with revised_tables and table_spec + end check + end for + return revised_tables +end action + +define action called migration_20200102000000 with parameters immutable_source: + store before_tables as historical_initial_tables + store after_tables as historical_csrf_tables + store before_sessions as before_tables[1] + store after_sessions as after_tables[1] + create new OrmMigration as migration_spec: + migration_id is "20200102000000" + migration_name is "Persist session CSRF tokens" + up_steps is [(orm_migration_add_column of before_sessions and after_sessions and (historical_default of "csrf_token" and ""))] + irreversible_reason is "removing csrf_token would discard live session security state" + managed_before is before_tables + managed_after is after_tables + source_text is immutable_source + end + return migration_spec +end action diff --git a/app/models.wfl b/app/models.wfl new file mode 100644 index 0000000..69d0db8 --- /dev/null +++ b/app/models.wfl @@ -0,0 +1,172 @@ +// Current application mapping. Historical migrations own their own immutable +// definitions; changing this file never changes an applied migration. +include from "migrations.wfl" + +define action called db_text_field with parameters field_name and nullable_value: + create new OrmField as field_spec: + field_name is field_name + value_type is "text" + nullable is nullable_value + end + return field_spec +end action + +define action called db_default_text_field with parameters field_name and default_text: + create new OrmField as field_spec: + field_name is field_name + value_type is "text" + has_default is yes + default_value is default_text + end + return field_spec +end action + +define action called db_timestamp_field with parameters field_name: + create new OrmField as field_spec: + field_name is field_name + value_type is "text" + nullable is yes + server_default is "current timestamp" + end + return field_spec +end action + +define action called db_generated_key: + create new OrmField as field_spec: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes + end + return field_spec +end action + +define action called db_reference_field with parameters field_name and nullable_value: + create new OrmField as field_spec: + field_name is field_name + value_type is "identity" + nullable is nullable_value + end + return field_spec +end action + +define action called db_users_model: + create new OrmField as username_spec: + field_name is "username" + unique_value is yes + end + create new OrmField as password_spec: + field_name is "password_hash" + sensitive is yes + end + create new OrmField as role_spec: + field_name is "role" + sensitive is yes + has_default is yes + default_value is "author" + end + create new OrmModel as model_spec: + table_name is "users" + fields is [(db_generated_key) and username_spec and password_spec and role_spec and (db_timestamp_field of "created_at")] + end + return model_spec +end action + +define action called db_sessions_model: + // SQLite's historical text primary key permits null. This declaration keeps + // that stored-data contract; the authentication API still requires text IDs. + create new OrmField as session_key: + field_name is "id" + primary_key is yes + nullable is yes + sensitive is yes + end + create new OrmField as csrf_spec: + field_name is "csrf_token" + sensitive is yes + has_default is yes + default_value is "" + end + create new OrmModel as model_spec: + table_name is "sessions" + fields is [session_key and (db_reference_field of "user_id" and no) and (db_timestamp_field of "created_at") and (db_text_field of "expires_at" and no) and csrf_spec] + end + create new OrmRelationship as user_relation: + relation_name is "user" + related_model is db_users_model + local_field is "user_id" + related_field is "id" + end + model_spec.relate(user_relation) + return model_spec +end action + +define action called db_content_model with parameters table_name: + create new OrmField as slug_spec: + field_name is "slug" + unique_value is yes + end + create new OrmModel as model_spec: + table_name is table_name + fields is [(db_generated_key) and slug_spec and (db_text_field of "title" and no) and (db_default_text_field of "body_markdown" and "") and (db_default_text_field of "status" and "draft") and (db_reference_field of "author_id" and yes) and (db_timestamp_field of "created_at") and (db_timestamp_field of "updated_at")] + end + create new OrmRelationship as author_relation: + relation_name is "author" + related_model is db_users_model + local_field is "author_id" + related_field is "id" + end + model_spec.relate(author_relation) + return model_spec +end action + +define action called db_settings_model: + create new OrmField as setting_key: + field_name is "skey" + primary_key is yes + nullable is yes + end + create new OrmModel as model_spec: + table_name is "settings" + fields is [setting_key and (db_default_text_field of "svalue" and "")] + end + return model_spec +end action + +define action called db_media_model: + create new OrmField as filename_spec: + field_name is "filename" + unique_value is yes + end + create new OrmField as size_spec: + field_name is "size" + value_type is "integer" + has_default is yes + default_value is 0 + end + create new OrmModel as model_spec: + table_name is "media" + fields is [(db_generated_key) and filename_spec and (db_default_text_field of "original_name" and "") and (db_default_text_field of "content_type" and "") and size_spec and (db_reference_field of "uploader_id" and yes) and (db_timestamp_field of "created_at")] + end + create new OrmRelationship as uploader_relation: + relation_name is "uploader" + related_model is db_users_model + local_field is "uploader_id" + related_field is "id" + end + model_spec.relate(uploader_relation) + return model_spec +end action + +define action called db_login_attempts_model: + create new OrmIndex as attempt_index: + index_name is "idx_login_attempts_ip_time" + field_names is ["ip" and "attempted_at"] + end + create new OrmModel as model_spec: + table_name is "login_attempts" + fields is [(db_generated_key) and (db_text_field of "ip" and no) and (db_timestamp_field of "attempted_at")] + indexes is [attempt_index] + end + return model_spec +end action diff --git a/app/persistence-helpers.wfl b/app/persistence-helpers.wfl new file mode 100644 index 0000000..281ab46 --- /dev/null +++ b/app/persistence-helpers.wfl @@ -0,0 +1,273 @@ +// Compatibility adapters keep the existing application's map projections and +// ordinary numeric IDs. The reusable ORM always treats identities as exact text. +include from "models.wfl" + +define action called db_identity with parameters raw_identity: + check if isnothing of raw_identity: + return nothing + end check + store identity_text as raw_identity + check if (typeof of raw_identity) is equal to "Number": + check if raw_identity is less than -9007199254740991 or raw_identity is greater than 9007199254740991 or (floor of raw_identity) is not equal to raw_identity: + call orm_fail with "application identity" and "use exact integer text for identifiers outside the safe numeric range" + end check + change identity_text to "" with raw_identity + end check + store problem_text as orm_identity_problem of identity_text + check if problem_text is not equal to "": + call orm_fail with "application identity" and problem_text + end check + return identity_text +end action + +define action called db_legacy_identity with parameters identity_text: + check if isnothing of identity_text: + return nothing + end check + store magnitude as identity_text + check if (substring of magnitude and 0 and 1) is equal to "-": + change magnitude to substring of magnitude and 1 and ((length of magnitude) minus 1) + end check + check if (orm_decimal_exceeds of magnitude and "9007199254740991") is equal to no: + return parse_json of identity_text + end check + return identity_text +end action + +// Preserve the old route fallback while avoiding floating-point parsing of IDs. +define action called db_route_identity with parameters identity_text: + check if (orm_identity_problem of identity_text) is not equal to "": + return 0 + end check + return db_legacy_identity of identity_text +end action + +define action called db_lookup with parameters model_spec and field_name and field_value: + store requested as orm_query of model_spec + // Compatibility wrappers historically used "= ?" (SQL NULL never matches). + // The reusable ORM deliberately gives explicit NULL equality IS NULL meaning. + check if field_value is nothing: + call orm_where with requested and (orm_any of []) + return requested + end check + call orm_where with requested and (orm_compare of field_name and "equal" and field_value) + return requested +end action + +define action called db_insert_result with parameters saved: + create map result_value: + "affected_rows" is 1 + "last_insert_id" is db_legacy_identity of saved.inserted_rowid + end map + return result_value +end action + +// These are the only application SQL-clock exceptions. SQLite remains the time +// authority; ordinary row reads/writes still use model queries and records. +define action called db_clock with parameters conn and modifier: + store session as orm_borrow of conn + store native_rows as orm_sql_query of session and "SELECT datetime('now', ?) AS timestamp_value" and [modifier] + store native_row as native_rows[0] + return native_row["timestamp_value"] +end action + +define action called db_current_year with parameters conn: + store session as orm_borrow of conn + store native_rows as orm_sql_query of session and "SELECT strftime('%Y','now') AS year_value" and [] + store native_row as native_rows[0] + return native_row["year_value"] +end action + +// Legacy list wrappers return their full requested window, using bounded ORM +// pages internally. Negative limit retains SQLite's historical unlimited form; +// application code uses explicit positive limits or -1 for its existing lists. +define action called db_read_records with parameters session and requested and wanted_count and offset_value and relation_name: + check if (typeof of wanted_count) is not equal to "Number" or (typeof of offset_value) is not equal to "Number": + call orm_fail with "application page" and "limit and offset must be whole numbers" + end check + check if (floor of wanted_count) is not equal to wanted_count or (floor of offset_value) is not equal to offset_value: + call orm_fail with "application page" and "limit and offset must be whole numbers" + end check + store next_offset as offset_value + check if next_offset is less than 0: + change next_offset to 0 + end check + store collected as [] + store remaining as wanted_count + store has_more as wanted_count is not equal to 0 + repeat while has_more: + store batch_limit as 1000 + check if remaining is greater than 0 and remaining is less than batch_limit: + change batch_limit to remaining + end check + call orm_page with requested and batch_limit and next_offset + store page_records as orm_find of session and requested + check if relation_name is not equal to "": + call orm_load with session and page_records and relation_name + end check + for each record_value in page_records: + push with collected and record_value + end for + change next_offset to next_offset plus batch_limit + check if remaining is greater than 0: + change remaining to remaining minus batch_limit + end check + change has_more to (length of page_records) is equal to batch_limit and remaining is not equal to 0 + end repeat + return collected +end action + +define action called db_delete_matching with parameters session and requested: + call orm_page with requested and 1000 and 0 + store affected_total as 0 + store last_identity as 0 + store has_more as yes + repeat while has_more: + store result_value as orm_delete of session and requested + change affected_total to affected_total plus result_value["affected_rows"] + change last_identity to result_value["last_insert_id"] + change has_more to result_value["affected_rows"] is equal to 1000 + end repeat + create map combined_result: + "affected_rows" is affected_total + "last_insert_id" is last_identity + end map + return combined_result +end action + +define action called db_user_map with parameters record_value and include_hash: + check if record_value is nothing: + return nothing + end check + check if include_hash: + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "username" is orm_get of record_value and "username" + "password_hash" is orm_get of record_value and "password_hash" + "role" is orm_get of record_value and "role" + "created_at" is orm_get of record_value and "created_at" + end map + return mapped + end check + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "username" is orm_get of record_value and "username" + "role" is orm_get of record_value and "role" + "created_at" is orm_get of record_value and "created_at" + end map + return mapped +end action + +define action called db_related_username with parameters record_value and relation_name: + store related_record as orm_related of record_value and relation_name + check if related_record is nothing: + return nothing + end check + return orm_get of related_record and "username" +end action + +define action called db_post_map with parameters record_value and projection_kind: + check if record_value is nothing: + return nothing + end check + store author_name as db_related_username of record_value and "author" + check if projection_kind is equal to "published": + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "slug" is orm_get of record_value and "slug" + "title" is orm_get of record_value and "title" + "body_markdown" is orm_get of record_value and "body_markdown" + "status" is orm_get of record_value and "status" + "created_at" is orm_get of record_value and "created_at" + "author_name" is author_name + end map + return mapped + end check + check if projection_kind is equal to "admin": + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "slug" is orm_get of record_value and "slug" + "title" is orm_get of record_value and "title" + "status" is orm_get of record_value and "status" + "author_id" is db_legacy_identity of (orm_get of record_value and "author_id") + "updated_at" is orm_get of record_value and "updated_at" + "author_name" is author_name + end map + return mapped + end check + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "slug" is orm_get of record_value and "slug" + "title" is orm_get of record_value and "title" + "body_markdown" is orm_get of record_value and "body_markdown" + "status" is orm_get of record_value and "status" + "author_id" is db_legacy_identity of (orm_get of record_value and "author_id") + "created_at" is orm_get of record_value and "created_at" + "updated_at" is orm_get of record_value and "updated_at" + "author_name" is author_name + end map + return mapped +end action + +define action called db_page_map with parameters record_value and projection_kind: + check if record_value is nothing: + return nothing + end check + check if projection_kind is equal to "navigation": + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "slug" is orm_get of record_value and "slug" + "title" is orm_get of record_value and "title" + end map + return mapped + end check + check if projection_kind is equal to "admin": + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "slug" is orm_get of record_value and "slug" + "title" is orm_get of record_value and "title" + "status" is orm_get of record_value and "status" + "updated_at" is orm_get of record_value and "updated_at" + end map + return mapped + end check + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "slug" is orm_get of record_value and "slug" + "title" is orm_get of record_value and "title" + "body_markdown" is orm_get of record_value and "body_markdown" + "status" is orm_get of record_value and "status" + "author_id" is db_legacy_identity of (orm_get of record_value and "author_id") + "updated_at" is orm_get of record_value and "updated_at" + end map + return mapped +end action + +define action called db_media_map with parameters record_value and include_uploader: + check if record_value is nothing: + return nothing + end check + check if include_uploader: + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "filename" is orm_get of record_value and "filename" + "original_name" is orm_get of record_value and "original_name" + "content_type" is orm_get of record_value and "content_type" + "size" is orm_get of record_value and "size" + "uploader_id" is db_legacy_identity of (orm_get of record_value and "uploader_id") + "created_at" is orm_get of record_value and "created_at" + "uploader_name" is db_related_username of record_value and "uploader" + end map + return mapped + end check + create map mapped: + "id" is db_legacy_identity of (orm_get of record_value and "id") + "filename" is orm_get of record_value and "filename" + "original_name" is orm_get of record_value and "original_name" + "content_type" is orm_get of record_value and "content_type" + "size" is orm_get of record_value and "size" + "uploader_id" is db_legacy_identity of (orm_get of record_value and "uploader_id") + "created_at" is orm_get of record_value and "created_at" + end map + return mapped +end action diff --git a/app/render.wfl b/app/render.wfl index 8ab3338..a304e04 100644 --- a/app/render.wfl +++ b/app/render.wfl @@ -16,8 +16,7 @@ include from "../lib/scribe/src/scribe.wfl" // Shared site context, present on every page as `site`. define action called site_context with parameters conn and user: store nav_pages as page_list_published of conn - store yr_rows as query conn with "SELECT strftime('%Y','now') AS y" - store yr_row as yr_rows[0] + store current_year as db_current_year of conn create map site_map: "title" is setting_get of conn and "site_title" and "Scriptorium" "tagline" is setting_get of conn and "site_tagline" and "Words, well kept." @@ -26,7 +25,7 @@ define action called site_context with parameters conn and user: "is_admin" is is_admin of user "csrf" is field_or of user and "csrf_token" and "" "asset_base" is "/assets" - "year" is yr_row["y"] + "year" is current_year end map return site_map end action diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4e3d27a..e3c3920 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -33,7 +33,11 @@ main.wfl ── listen on port ── main loop: wait for request ── route Library layers, each pulled in with `include from` (see the include rule below): ``` -render.wfl ── auth.wfl ── db.wfl ── util.wfl +render.wfl ── auth.wfl ── db.wfl ── persistence-helpers.wfl ── models.wfl + └─ migrations.wfl + ├─ util.wfl + ├─ lib/orm/migrations.wfl (linear ORM chain) + └─ immutable app/migrations versions └──────── lib/scribe/src/scribe.wfl (git submodule) site_ext.wfl ── render.wfl (the site-extension seam) main.wfl ── site_ext.wfl (+ defines the router and every request handler) @@ -43,10 +47,15 @@ main.wfl ── site_ext.wfl (+ defines the router and every request handler) `file_stem`, `config_value_from`, `install_validate`. Form/cookie parsing is *not* here: WFL's stdlib already ships `parse_form_urlencoded` and `parse_cookies` (both percent-decode), so we use those. -- **db.wfl** — the SQLite schema (idempotent `CREATE TABLE IF NOT EXISTS`) and - every `query`/`execute` the app runs. Helpers take the connection handle as a - parameter, so the data layer is testable against `sqlite::memory:`. Includes - the installer lock (`install_is_done` / `install_mark_done` / `install_apply`). +- **db.wfl** — compatibility actions backed by ORM models, records and queries. + Helpers retain the caller-owned connection and existing result maps. The + installer commits its first user and settings atomically. The only specialized + application SQL reads SQLite's clock/year in `persistence-helpers.wfl`. +- **models.wfl** — the current mapping for all seven managed tables. + **migrations.wfl** and immutable version modules own schema, legacy adoption + and the migration ledger. Startup applies supported upgrades; the administrative + WFL entry point plans changes and performs explicit rollback. See + [ORM API](orm.md) and [migration operations](migrations.md). - **auth.wfl** — `hash_password`/`verify_password`, session create/lookup/delete (each session carries a CSRF token, validated by `csrf_ok`), and the `is_admin` / `can_edit` role checks. @@ -127,15 +136,19 @@ noted inline: 2. **Includes form a tree, not a flat namespace — diamonds break.** A file only sees definitions from files *it* includes (transitively), and an include that was already pulled in elsewhere is skipped. → The libraries form a strict - chain `util ← db ← auth ← render`, `main` includes only `render`, and the + chain shown above, `main` includes only `site_ext`, and the router + handlers live in `main.wfl` (so they share one scope). Scribe is included exactly once (by `render.wfl`). 3. **No query strings.** *Lifted in 26.7.26* (`query` / `query of req` + `parse_query_string`). Scriptorium's URLs still keep state in the path (`/blog/page/2`) — they predate the fix and are stable, shareable URLs. -4. **No transactions / no migration engine.** → Schema is idempotent DDL run at - boot; every write is a single statement. (The one post-MVP column addition, - `sessions.csrf_token`, is a `try`-guarded `ALTER TABLE` at boot.) +4. **Transactions and migration ownership.** Native WFL transaction blocks pin + SQLite connections and roll back errors. The reusable WFL ORM and versioned + migration engine now own schema evolution; startup inspects and adopts + recognized legacy shapes, including sessions without CSRF. Schema + transactions require the upstream runtime prerequisites documented in + [runtime capabilities](runtime-capabilities.md). No broad catch treats + migration failure as success. 5. **No CSRF/session helpers.** → Sessions are a random `secure_random_bytes` id in an `HttpOnly; SameSite=Lax` cookie, stored in a `sessions` table with a SQL-checked expiry. CSRF tokens ride the same table — see Security below. diff --git a/docs/independent-migration-review.md b/docs/independent-migration-review.md new file mode 100644 index 0000000..3585b22 --- /dev/null +++ b/docs/independent-migration-review.md @@ -0,0 +1,81 @@ +# Independent migration and schema review + +Technical review on 2026-09-20, independent of the migration implementation +owner. This is not Maintainer approval or final remote CI acceptance. + +The review covers historical schema declarations, migration fingerprints and +history replay, legacy adoption, explicit rebuild copy maps, native transaction +ownership, reversible-range checks, CLI scaffolding and target selection, and +concurrent migration execution. ORM record and application compatibility review +is recorded separately in [orm-independent-review.md](orm-independent-review.md). + +## Findings and remedies + +| Finding | Executable evidence and reviewed remedy | +| --- | --- | +| SQL normalization erased meaningful quoted identifier whitespace | A table containing `"ti tle"` was accepted as the declared `title` column. The unchanged WFL assertion is retained in `TestPrograms/migration-schema-safety.test.wfl`. Normalization now tokenizes SQL, preserves literal bytes and identifier content, and only removes ordinary identifier quoting. | +| An incomplete rebuild copy map silently lost a retained nullable field | A same-schema rebuild copying only `id` changed an existing title to NULL and reported success. The retained WFL regression now requires refusal and the original value. Rebuilds require unique source/destination mappings and explicit same-name copies for every retained field. Removed or renamed fields remain explicit; unrecoverable data loss requires an irreversible declaration in either direction. | +| Tables appearing only in a historical before-schema escaped drift checks | A declared dropped table could remain present without detection. The retained WFL regression now requires refusal. The managed table universe includes both before and after declarations. | +| A concurrent runner could move beyond the requested target while the waiting runner reported success | Source review found the old greater-than/less-than completion conditions. Up and down now finish only at equality, reject movement beyond the target or opposite to their observed direction, and re-read history and schema under each native write lock. Status and plan use one read snapshot. Real process coverage is in `tests/integration/migrations-recovery.test.wfl`. | +| A historical managed index recreated after rollback escaped drift checks | A real version-two to version-one rollback followed by recreating `idx_items_title` still allowed status to succeed. `TestPrograms/migration-index-safety.test.wfl` preserves the independently confirmed failing assertion. The remedy tracks historical managed index names from both before and after declarations and rejects their unexpected presence, without claiming unknown extension indexes. SQLite object identities use consistent ASCII case-insensitive catalog lookup and folded managed-name sets. | + +The first three assertions ran as meaningful behavioral failures before the +remedies: the initial local probe reported **0/3**, including an observed title +value of NULL after the unsafe rebuild. The unchanged assertions then passed +**3/3**, with the original title retained. Local logs are +`target/migration-review-red.txt` and `target/migration-review-green.txt`. +The scenarios were promoted to `TestPrograms/migration-schema-safety.test.wfl`. +Parser or static-analysis diagnostics are not counted as the behavioral Red. + +The index regression independently reported **0/1**, exit 1, against Scriptorium +`9de5c72` and the combined runtime below. The failure is the actual assertion +that status should refuse the incompatible schema. Local log: +`target/migration-index-review-red.txt`. It is retained unchanged apart from its +include path when moving from the ignored probe directory to `TestPrograms`. + +The index Red is preserved in commit `af2b6ac`; its Green remedy is `e3e8a6a`. +After the remedy, independent +execution against the same combined runtime passed the index suite **1/1** and +the original schema safety suite **3/3**, both exit 0. The index test also checks +uppercase spelling, with its paired DROP/CREATE fixture mutations inside one +native schema transaction. Logs: `target/migration-index-final-review.txt` and +`target/migration-schema-final-review.txt`. Source re-review found no remaining +blocking migration/schema finding. + +The implementation owner's final real-process recovery result is **3/3**, and +the CLI suite is **4/4**. Their logs were inspected at +`target/capability-probes/migrations-recovery-final.log` and +`target/capability-probes/migrations-final.log`; these are owner-run evidence, +distinct from the independent safety reruns above. + +## Review boundaries + +Schema checks deliberately compare conservative SQL tokens rather than claiming +a complete SQLite SQL equivalence parser. Unsupported declarations are refused. +The reviewed rebuild performs its row copy inside SQLite, retains sequence +values as exact decimal text, recreates extension indexes and triggers, and +refuses ambiguous column removal in the presence of extension triggers. Native +schema transactions perform foreign-key checks before commit and restore +connection settings; their implementation and cancellation review is recorded +in [runtime-review.md](runtime-review.md). + +The write lock covers one version, not an entire multi-version command. A later +failure can leave earlier successfully committed versions applied. The concurrent +target test accepts either an initial-plan target refusal or a locked recheck +concurrency refusal, depending on scheduling; it does not claim to prove one +particular lock position with a fixed delay. Both must return a nonzero result. + +During the additional uppercase fixture check, separate unpinned pooled DROP +and CREATE statements exposed a stale-schema preparation symptom: CREATE said +the index existed, while an intervening catalog SELECT confirmed it had been +dropped and allowed the same CREATE. No delay or error suppression was added. +The fixture now uses the same pinned schema scope as production migrations for +the paired edits. This review does not establish general atomicity or schema +cache coherence for a sequence of independent raw SQL calls on different pooled +connections. + +The local combined runtime is WFL source `df6ad9a2`, executable SHA-256 +`b96c06f6013a9a64d4d042c9b379a30af1c8d274d7855b5ebe9d7fe14a750d1c`. +It is an integration candidate, not evidence that the published nightly already +contains the upstream remedies. The complete suite and exact-revision remote +Linux/Windows and resolved-image evidence remain separate acceptance requirements. diff --git a/docs/migration-evidence.md b/docs/migration-evidence.md new file mode 100644 index 0000000..e16963f --- /dev/null +++ b/docs/migration-evidence.md @@ -0,0 +1,105 @@ +# Migration implementation evidence — 2026-09-20 + +This records focused migration evidence, separate from the repository's full +WFL, HTTP, tooling and pinned Scribe gates. The canonical behavior and operator +workflow are in [migrations.md](migrations.md). + +## Runtime and scope + +Final focused execution used the integrated WFL candidate at +`target/runtime/combined-df6ad9a2/wfl.exe`, SHA-256 +`b96c06f6013a9a64d4d042c9b379a30af1c8d274d7855b5ebe9d7fe14a750d1c`. +It includes the reviewed native application-error/schema-transaction, HTTP +response and process-control prerequisites. The stock official 26.9.12 runtime +predates those capabilities; see [runtime review](runtime-review.md) for upstream +PR and gate evidence. This record does not itself claim a merged/published +nightly or deployment approval. + +All new scenarios, fixture generation, assertions, process ownership and cleanup +are WFL. Tests use synthetic file-backed SQLite databases under ignored +`target/test-artifacts`, with native handles and owned child processes closed in +`finally`. No real site database, upload, password or token was used. + +## Red and independent review + +Commit `72a2d6d` preserves the initial behavioral Red. The prior `db_migrate` +retained a sessions-only legacy row, its exact 64-bit reference, NULL timestamp +and added CSRF default, but did not create migration history. The test failed +at the ledger assertion: expected one history table, observed zero. Its log is +`target/capability-probes/migration-adoption-red.log`; the unchanged acceptance +test passes through the new application compatibility wrapper. + +Independent review found three additional concrete schema defects, reproduced +as WFL assertion failures in `target/testing-probes/migration-review.test.wfl` +and `target/migration-review-red.txt`: + +1. Removing spaces from quoted identifiers could confuse `"ti tle"` with + `title` and falsely accept drift. +2. A rebuild copy map could omit a retained nullable column and lose its value. +3. The managed object universe omitted tables declared only in `managed_before`. + +Those cases are preserved in `TestPrograms/migration-schema-safety.test.wfl`. +Token-aware inspection, complete retained-field mappings and the union of all +historical before/after declarations pass all three. Independent re-review and +rerun also passed. A further review of target races added strict overshoot and +opposite-direction checks. Read-only plans now use one native transaction +snapshot to avoid mixing ledger/schema states across another process's commit. + +The conflicting-target process test intentionally accepts refusal in either +the initial consistent plan or the locked recheck. It does not infer a specific +lock position from a sleep. Two real same-target runners are separately required +to both succeed with exactly one event per migration. + +A final independent Red, committed as `af2b6ac`, found that a known historical +managed index could be reintroduced after rollback without being reported as +drift. `TestPrograms/migration-index-safety.test.wfl` preserves that case. The +managed index universe now includes every historical before/after declaration +and checks expected absence as well as presence. SQLite's ASCII-insensitive +object identity is respected, including an uppercase reintroduced name. The +regression passes on the corrected source. + +## Final focused result + +Each suite below was invoked as `wfl --test ` using the candidate above. +All nine suites passed on final source: **24 tests, zero failures**. + +| WFL suite | Passed | Observed contract | +| --- | ---: | --- | +| `TestPrograms/migration-adoption.test.wfl` | 1 | Supported sessions-only upgrade, exact values and audited history | +| `TestPrograms/migration-engine.test.wfl` | 3 | Read-only plan, targeted/repeated apply, rollback/reapply events, full-range irreversible preflight, changed/missing history and drift | +| `TestPrograms/migration-failures.test.wfl` | 3 | SQLite read-only error, invalid registry/scaffold/destructive reversal, inaccessible child path | +| `TestPrograms/migration-legacy-matrix.test.wfl` | 3 | Both complete historical schemas, current sessions-only state, preserved seven-table data and extensions, refusal of unknown partial/altered schemas | +| `TestPrograms/migration-rebuild.test.wfl` | 3 | Exact 64-bit high water including an empty table, cascading children, extension index/trigger, failed rebuild rollback and retry | +| `TestPrograms/migration-schema-safety.test.wfl` | 3 | Independently reproduced inspection, copy-map and prior-only object regressions | +| `TestPrograms/migration-index-safety.test.wfl` | 1 | Reintroduced historical managed index after rollback, including ASCII case variation | +| `tests/tooling/migrations.test.wfl` | 4 | Real CLI exit codes, absent/configured/overridden targets, read-only legacy target planning, source edit detection, completed scaffold registration and actual up/down execution | +| `tests/integration/migrations-recovery.test.wfl` | 3 | Five-second contention bound, killed owner rollback, same-target convergence, conflicting target refusal | + +Final logs are ignored `target/capability-probes/*-final.log`. The SQLite +read-only scenario opens a native `file:/absolute/path/site.db?mode=ro` URI, +so every pooled connection has the read-only restriction. +It checks a genuine SQLite read-only failure without depending on whether a +privileged CI user bypasses OS file permissions. It is not claimed as a +cross-platform ACL test. The lock test asserts elapsed time is +at least four seconds and less than eight around the native five-second bound. + +The killed owner has entered a real schema scope, written a schema object, +record and uncommitted event, and signaled readiness before the parent terminates +and reaps it. The parent verifies that all three writes disappeared, history +remains valid, and a later migration succeeds. Separate upstream WFL tests cover +cancelled handler futures, process exit and ordinary-return compatibility. + +Two fixture assumptions were corrected after failed executions. A preliminary +`PRAGMA query_only=ON` fixture could affect only one pooled connection; a rerun +observed three applied versions instead of the expected two. The native read-only +URI replaces that assumption without changing the migration assertion. Also, +bare pooled DROP/CREATE of the same synthetic index could report stale schema +during preparation; querying SQLite proved the DROP had occurred. That paired +fixture DDL now uses the documented native schema scope, with no arbitrary +sleep or retry. Production migration steps already use that pinned scope. + +The full HTTP suite independently tests a stopped-site database-and-uploads +backup restore and a pre-CSRF installed site's extension/theme/login behavior. +Those results belong to the HTTP evidence record. Full repository discovery, +Linux/Windows CI and the final published-runtime verification remain the root +task's gates; focused local results do not substitute for them. diff --git a/docs/migrations.md b/docs/migrations.md new file mode 100644 index 0000000..4dd45f2 --- /dev/null +++ b/docs/migrations.md @@ -0,0 +1,304 @@ +# Versioned SQLite migrations + +Scriptorium runs immutable historical migrations at startup through +`scriptorium_migrate(conn)`. The compatibility entry point `db_migrate(conn)` +still returns `yes` and leaves the caller's native handle open. Current model +declarations never generate startup schema changes. Change the schema by adding +a migration, then update application models and queries in the same reviewed +change. + +This implementation requires WFL's `raise_error` and +`in transaction on connection for schema changes` capabilities. Official +26.9.12 predates those prerequisites; official nightly 26.9.14 at source +`8d82d785` contains them. See [runtime verification](orm-verification.md) for +immutable publication provenance and [runtime review](runtime-review.md) for +the upstream safety evidence. An older installed executable that parses +ordinary transactions is insufficient. + +## Commands and target selection + +Run from the application root, as for `main.wfl`: + +```text +wfl scripts/migrate.wfl status +wfl scripts/migrate.wfl plan +wfl scripts/migrate.wfl up +wfl scripts/migrate.wfl up --target 20200101000000 +wfl scripts/migrate.wfl down --count 1 +wfl scripts/migrate.wfl down --target 20200101000000 +wfl scripts/migrate.wfl new add_post_summary +``` + +Every database command prints its selected target. The CLI uses the same +`.wflcfg` parser and `data_dir` rule as startup: an unset/empty directory selects +`./scriptorium.db`; a configured directory selects `/scriptorium.db`. +The first exact matching configuration key wins, and inline comments remain +part of the value, matching the application. `--database PATH` is an explicit +CLI override; it does not rewrite application configuration. Relative paths +resolve from the application root/current working directory. + +`status` and `plan` inspect without applying or adopting anything. If the target +file is absent they report the fresh plan without opening SQLite or creating +directories. Existing databases are inspected with a consistent read +transaction. A legacy plan states the recognized adoption version and the +number of migrations up to the requested target. A target older than the +recognized legacy schema is refused. + +`up` creates a missing database parent directory, performs supported legacy +adoption if necessary, and applies through the exact requested target (default +`latest`). It refuses a target older than the current version. `down` requires +exactly one of a positive `--count` or an exact `--target`; `zero` means no +applied migrations. Unknown targets, changed history, malformed arguments, +schema drift, locks, access failures and migration errors produce a nonzero +process exit and an actionable cause. A successful exit reports the applied and +pending versions. + +The two historical application migrations are deliberately irreversible: + +| ID | Historical change | Why rollback is refused | +| --- | --- | --- | +| `20200101000000` | Original seven application tables and rate-limit index | Dropping them destroys site records | +| `20200102000000` | Add `sessions.csrf_token` with empty-text default | Dropping it discards live session security state | + +The engine supports reversible later migrations. It preflights the complete +requested rollback range before changing any version. A reversible index change +above an irreversible baseline is not partially rolled back when the requested +range also includes that baseline. Recreating an empty table or adding an empty +column is not restoration of its previous data. + +## Legacy adoption + +The accepted legacy inputs are intentionally limited to the inspected +[persistence inventory](persistence-inventory.md): + +- No application-managed tables: apply the initial schema and CSRF migration. +- All seven original tables and the rate-limit index, without CSRF: adopt the + first version, then apply the CSRF migration. +- All seven current legacy tables and the rate-limit index, with CSRF: adopt + both versions without rewriting records. +- Only the exact original sessions table, with or without CSRF: preserve it, + create the missing historical tables, and adopt the matching version before + applying any later versions. This preserves the old supported partial state. + +Other partial states, missing managed indexes, altered constraints or unknown +managed columns are refused before adoption writes. Inspection compares schema +tokens conservatively: whitespace and ASCII case outside literals, and quoting +of ordinary identifiers, are insignificant; literal bytes, identifier content, +column order, constraint clauses, collation, generated expressions, STRICT and +WITHOUT ROWID remain significant. Semantically equivalent but differently +authored constraint orders can be refused; they need deliberate review, not +blind stamping. + +Adoption preserves IDs (including exact signed 64-bit identities), hashes, +session tokens, timestamps, NULLs, empty text, historical defaults, custom roles +and statuses, and orphan references permitted by the original schema. It does +not introduce foreign keys into application tables. Unmanaged extension tables, +indexes and triggers stay outside core history and are retained. + +## Authoring a migration + +`new ascii_name` creates a timestamped file under `app/migrations` and prints the +exact include and registry changes required. It never opens a database. The +generated declaration is deliberately incomplete: an empty `up_steps` list is +rejected. Complete and test the declaration before registering it. + +WFL currently has no namespaced dynamic-module export mechanism. Versions +therefore form a static include chain: each new historical module includes its +predecessor, the application's migration bridge includes the newest module, and +`scriptorium_migration_registry` returns each version once in ascending order. +Replace the bridge's final historical include; do not add an include diamond. +The scaffold test completes, registers, applies and rolls back a generated +version, so the parent-provider reference is exercised through real execution. + +An `OrmMigration` carries: + +| Field | Meaning | +| --- | --- | +| `migration_id` | Unique increasing 14-digit UTC timestamp text | +| `migration_name` | Stable human-readable description | +| `managed_before`, `managed_after` | Complete historical managed table declarations at that version | +| `up_steps`, `down_steps` | Ordered typed steps, with an explicit SQL escape hatch | +| `irreversible_reason` | A specific explanation when data cannot be restored | +| `source_text` | Entire immutable version source, read from its registered file | + +Each version declares historical `OrmModel`/`OrmField`/`OrmIndex` values wrapped +in `OrmSchemaTable`; it must not import or call the application's current model +registry. Consecutive after/before schemas must match. `OrmForeignKey` describes +physical foreign keys separately from the record ORM's logical relationships. +Its local/target field lists are explicit; supported actions are `NO ACTION`, +`RESTRICT`, `CASCADE`, `SET NULL` and `SET DEFAULT`. Composite foreign keys are +supported; the record model still requires a single primary key. + +Schema helpers produce steps: + +```wfl +store create_step as orm_migration_create_table of historical_table +store add_step as orm_migration_add_column of old_table and new_table and added_field +store index_step as orm_migration_create_index of new_table and declared_index +store undo_index as orm_migration_drop_index of new_table and declared_index +store rebuild_step as orm_migration_rebuild of old_table and new_table and ["id", "title"] and ["id", "title"] +store data_step as orm_migration_sql of "UPDATE settings SET svalue=? WHERE skey=?" and ["new value", "setting name"] +``` + +`orm_migration_drop_table` requires an irreversible reason in either direction. +Discarding column values in either direction also requires a reason; an explicit +lossless rename copy preserves the values. Rebuilds can alter column types, +nullability/defaults and other supported declarations; a transformation outside +the typed vocabulary must be explicit authored SQL with runtime values bound as +parameters. Review SQL steps for loss of data and honest reversibility. Do not +put user input in identifiers, schema SQL, defaults or migration names. + +The reusable library exposes `orm_migrate(session, registry, target)`, +`orm_migration_status(session, registry)`, +`orm_migration_plan(session, registry, target)`, +`orm_rollback_count(session, registry, count)` and +`orm_rollback_to(session, registry, target)`. Targets are exact IDs, `latest` or +`zero`. These actions own their native transaction scopes and must be called +outside another transaction. A borrowed session leaves connection ownership +with its caller. + +## History, atomicity and concurrency + +`_orm_migrations` records the currently applied contiguous prefix, names, +checksums and timestamps. `_orm_migration_events` retains ordered apply, adopt +and rollback events even when a version is no longer applied. History validation +replays those events and checks the current ledger against the result; missing +files, edited sources, mismatched names, missing history tables and inconsistent +event order fail before execution. The `_orm_` prefix is reserved. + +The `orm-migration-v1` checksum uses SHA-256 over deterministic ordered lists +containing the entire descriptor, typed schema and steps, explicit SQL and bound +values, reversibility reason, and entire source. Only CRLF becomes LF in source +text; other whitespace and comments participate. JSON map key iteration is not +used. Do not edit an applied or previously rolled-back version. Restore its +original source and append a new version. Preserve this fingerprint format when +evolving the library so existing history stays verifiable. + +Each version's schema/data changes and ledger/event update commit together in a +native schema transaction. WFL pins one SQLite connection, disables foreign-key +enforcement before acquiring `BEGIN IMMEDIATE`, bounds connection/lock acquisition +at five seconds, checks every foreign key before commit, and restores enforcement +before reusing the connection. Errors roll back; `return no` is ordinary WFL +success and must never be used as an abort. Use `raise_error` for an actual +application validation failure. + +The write lock covers one migration, not the whole multi-version command. The +engine re-reads history and schema under every acquired lock. Two same-target +writers converge without duplicate history; a writer that observes history +beyond its target or moving in the opposite direction raises a concurrency +error. Review status before retrying a conflict. Earlier committed versions +remain committed if a later version fails. Status/plan use one read snapshot to +avoid combining a pre-commit ledger with a post-commit schema. + +## Rebuild and recovery + +Rebuild copies run entirely inside SQLite, without converting rows or sequence +values through WFL floating-point numbers. Every retained field must have an +explicit same-name copy mapping; source and destination lists cannot repeat +fields. The table's AUTOINCREMENT high-water mark is preserved as exact decimal +text even when the highest row was deleted. The engine recreates declared +indexes and existing extension indexes/triggers from their stored SQL. It +refuses column removal while extension triggers still exist because SQLite can +accept a trigger that only fails later when it refers to a removed column; +author an explicit reviewed trigger migration first. + +Native schema transactions preserve referencing rows while rebuilding a parent +table, including `ON DELETE CASCADE` children. Foreign-key validation before +commit catches invalid results and rolls back schema, data and history together. +An abandoned `_orm_rebuild_...` name is refused rather than guessed away. + +Before a production migration, stop the application and its writers, then back +up the whole configured data directory: database, SQLite sidecars if present, +and matching uploads. With the legacy layout, back up `scriptorium.db` and its +sidecars together with `static/uploads`. Keep the matching application and +migration sources with that backup. Restore while the app is stopped; do not +combine a database from one point in time with uploads from another. Verify +`PRAGMA integrity_check`, `PRAGMA foreign_key_check`, migration status, login, +content and uploads before reopening traffic. An irreversible migration calls +for that matching backup, not ledger deletion or an edited checksum. + +## Deployment and application-version compatibility + +Treat the application revision, immutable migration sources, WFL runtime and +site configuration as one release. Keep those exact versions, custom themes +and extension files with the matching database-and-uploads backup. A database +backup alone cannot reconstruct deployment configuration or extension code. + +1. Stop the application, extension writers and other migration runners. Record + the current revision, runtime and `.wflcfg`, then make the stopped-site backup + described above. Keep that backup separate from the working data directory. +2. Put the reviewed new application and migration sources in place and select + the required WFL runtime. From that application's root, run + `wfl scripts/migrate.wfl status` and `wfl scripts/migrate.wfl plan`. Check the + printed database target against the intended site's configuration. Inspect + the planned versions and any irreversible steps before proceeding. +3. Run `wfl scripts/migrate.wfl up`, or `up --target ID` when deploying a reviewed + intermediate version, then run `status` again. An intermediate schema must + match the application being deployed; normal application startup applies + every pending version in that application's registry. +4. Check integrity, foreign keys and the site's retained data. Start the app + with traffic held back, verify login, installation lockout, content, uploads, + theme and extension behavior, then reopen traffic. + +Startup supports fresh initialization and the legacy states listed above. It +does not promise that older application code can use a newer schema. Code with +this migration engine rejects history absent from its registry; older code +without that check must not be used to bypass it. Keep the deployed code's +registry and the database history together, including rolled-back source files +needed to validate migration events. + +For an application downgrade, restore the older release and its matching backup +unless that exact older-code/current-data combination has been reviewed and +tested. A schema rollback alone does not authorize switching to an older +checkout: rollback events still require the newer immutable migration files, +and normal startup with that complete registry reapplies pending versions. + +For a reviewed schema rollback while the application remains stopped, use the +**newer release's complete registry** to inspect and, only when every affected +migration is reversible, run `wfl scripts/migrate.wfl down --target ID`. Any +subsequent application release must explicitly support that schema, complete +event history and startup behavior. The two shipped historical migrations are +irreversible, so removing either requires restoration of a matching older +backup. If rollback is irreversible or fails preflight, keep the application +stopped and choose a reviewed forward repair or restore matching code, runtime, +configuration, database and uploads. Never delete ledger rows or edit checksums +to make older code start. + +## Interrupted-upgrade recovery + +1. Keep traffic closed and stop all application and migration writers. Preserve + the failed command's diagnostics and the current database with any sidecars; + do not delete sidecars or rebuild a table manually. Use the same immutable + release sources, runtime, configuration and intended target as the interrupted + attempt. +2. Run `wfl scripts/migrate.wfl status` and `wfl scripts/migrate.wfl plan`, checking + the printed database target again. SQLite recovers an uncommitted transaction + when the database is reopened. Each version must be either fully committed + with its ledger entry or absent with no partial schema/data changes. A commit + may have finished before command output was lost; use inspected history to + decide what remains. Inspect integrity and foreign keys as well. +3. If history and schema validation succeed, rerun `up` with the original + intended target. Applied versions are retained and only pending work runs. + A lock conflict requires confirming the other owner has stopped before + retrying. A checksum, history, schema or integrity failure requires diagnosing + the cause or restoring a backup; do not force adoption or rewrite history. +4. After a successful retry, repeat deployment verification before reopening + traffic. If restoring instead, stop all owners, replace the entire working + data set with the complete backup (including matching uploads and sidecars), + and restore its matching code, runtime and configuration. Remove the failed + working data from the restore destination first so stale files cannot mix + with the backup. For the legacy layout, replace the database and sidecars + together with `static/uploads`. For a versioned release, recheck migration + status, integrity, foreign keys and HTTP workflows before reopening traffic. + A restored pre-ORM release has no migration CLI: use that release's database + integrity and application verification procedure with its matching runtime. + +The WFL suites exercise real file-backed databases, all supported legacy states, +edited/missing history, drift and unsafe copy maps, rollback/reapply, whole-range +irreversibility checks, SQLite read-only URI access, bad paths, a genuine +five-second lock, concurrent processes and killed transaction-owner recovery. +CLI tests invoke real child processes, including generated-source execution. +HTTP recovery tests stop and restore a complete disposable site with matching +uploads. Runtime cancellation and process-exit cleanup have additional upstream +WFL coverage. Read-only URI coverage is portable even for privileged CI accounts; +it does not claim OS ACL or permission testing on every host. diff --git a/docs/orm-acceptance-audit.md b/docs/orm-acceptance-audit.md new file mode 100644 index 0000000..98f4322 --- /dev/null +++ b/docs/orm-acceptance-audit.md @@ -0,0 +1,58 @@ +# ORM acceptance audit + +This audit maps the requested deliverable to implementation and executable +evidence. `orm-verification.md` records the verified official runtime and the +isolated complete-runner CI failure proof. The final clean-head result and exact +runtime/source provenance are recorded in the Validation section of +[PR #16](https://github.com/WebFirstLanguage/Scriptorium/pull/16); local candidate +passes do not substitute for that resolved official-image gate. + +| Requirement | Implementation and executable evidence | +| --- | --- | +| Read governing policies, foundations and existing architecture | `orm-migrations-design.md`, `persistence-inventory.md`, `wfl-test-inventory.md`, `runtime-capabilities.md` record the sources and resolved attachment limitation | +| Verify real runtime capabilities | Five WFL suites in `tests/runtime/`; initial Blacksmith Red in `a81ecf9`; separate reviewed upstream HTTP, process and schema-transaction PRs | +| One progressive API and explicit migrations from the beginning | `examples/orm/progression.test.wfl` passes first-record, composed-query/relationship/transaction and reversible-index examples; `orm.md` maps the No-Unlearning Invariant | +| Models, types, defaults, nullable fields, identifiers and indexes | `lib/orm/types.wfl`, `models.wfl`, `schema.wfl`; `orm-types` and `orm-models` suites | +| Missing, NULL, empty and not-found semantics | `records.wfl`, `queries.wfl`; `orm-records`, `orm-crud`, `db-contracts` suites | +| Unknown and protected mass-assignment rejection | Atomic assignment validation and explicit single-field trusted setters/updates; `orm-records`, `orm-write-review` | +| Database constraints remain authoritative | INSERT/UPDATE and conflict-targeted upsert use SQLite constraints; `orm-crud` duplicate/unique cases; no check-then-insert uniqueness shortcut | +| CRUD, composed Boolean/NULL filters, ordering, pages, count and existence | `predicates.wfl`, `queries.wfl`, `writes.wfl`; `orm-predicates`, `orm-crud`, `orm-query-order` | +| Values parameterized, identifiers and directions validated | Predicate/query/schema constructors; malicious-value and identifier cases in `orm-types`, `orm-predicates`, `orm-write-review` | +| Guard bulk writes and preserve their selected page | Explicit whole-table intent, bounded physical row selection for nullable keys; independent Red `c0d0603`, Green `db-review-contracts` | +| Explicit batched relationships and visible cost | `relationships.wfl`; 101 parents use two queries; 1001 children use two; 10001 children fail without partial state; `orm-relationships`, `orm-relationship-limits` | +| Ownership, rollback, nesting and reusable connections | `connections.wfl`, native WFL transaction scopes; `orm-crud` rollback, nested rejection, post-failure reuse and borrowed-close cases | +| Actionable errors without assigned secrets | `orm_fail`, bound-text redaction, no parameter-value trace; `orm-records` and ownership/diagnostic cases | +| Explicit specialized SQL | `orm_sql_query` / `orm_sql_execute`; application exceptions are SQLite clock/year reads, described in `orm.md` | +| Scaffold/status/plan/up-target/down-count/down-target | `scripts/migrate.wfl`, typed engine and static immutable registry; `tests/tooling/migrations.test.wfl`, `migration-engine` | +| Ordered immutable checksums and audit history | `_orm_migrations` plus append-only `_orm_migration_events`; registry/order/checksum/history/schema validation in `migration-engine`, `migration-failures` | +| Serialize runners, bounded locks and atomic ledger changes | Native schema transactions with locked reinspection; real child processes in `tests/integration/migrations-recovery.test.wfl` | +| Interrupted work cannot claim success | Killed schema owner, lock failure, restart and same-target/opposing-target runners; `migrations-recovery` | +| SQLite rebuild preserves records and extension objects | Explicit column mapping, sequence high-water preservation, indexes/triggers/FK checking; `migration-rebuild`, independent `migration-schema-safety` | +| Reject fake or impossible rollback before changing anything | Irreversible declarations, complete-range preflight and retained-column copy checks; `migration-engine`, `migration-failures`, `migration-schema-safety` | +| Fresh initialization and safe legacy adoption | Immutable initial/CSRF versions; supported full pre-CSRF/current and sessions-only variants; `migration-adoption`, `migration-legacy-matrix` | +| Stop on ambiguous states and preserve unmanaged objects | Full managed declaration checks, conservative unsupported-partial rejection; `migration-legacy-matrix`, extension rebuild cases; `migration-index-safety` rejects reintroduced historical managed indexes | +| Target path, data directory, absent plan, read-only and invalid path | Shared config parser; no absent plan file creation; native SQLite read-only URI restricts every pooled connection; `tests/tooling/migrations`, `migration-failures` | +| All seven tables use ORM; no competing schema owner | `app/models.wfl`, `app/db.wfl`, `app/migrations.wfl`; ordinary data access has no raw row SQL | +| Preserve IDs, hashes, sessions, CSRF, timestamps, defaults and aliases | Existing WFL application suites, `db-contracts`, independent `db-review-contracts`; exact high IDs and atomic installer have retained behavioral Red | +| Preserve installation, routes, port, themes and hooks | Real WFL HTTP suites; `server-port`, `authentication`, `legacy-upgrade` checks locked upgraded installer and custom extension/theme across restart | +| Exercise content/users/settings/media/throttle | `content-users`, `media`, `throttle` HTTP suites with database assertions and both upload layouts | +| Verified database-plus-uploads backup/restore | `tests/integration/recovery.test.wfl` stops the owner, copies complete state, restores, then checks HTTP/auth/upload/integrity/FK behavior | +| Every test, fixture, helper and driver is WFL | Python runner, two Python tooling suites and Python port suite removed; WFL 8+28 behavior mapping plus strengthened diagnostics and default-discovery cases | +| Non-test Python checker remains only as a subject | `scripts/check_repo_hygiene.py` is invoked by WFL hygiene assertions and the existing governance utility step | +| Complete suite includes pinned Scribe | `wfl --execution-timeout 1200 scripts/run_tests.wfl` discovers application, tooling, integration, runtime probes, examples and an isolated copy of pinned Scribe; official nightly 26.9.16 includes the required invocation-budget option | +| Timeout/failure cleanup and failure propagation | Owned child lifecycle, joined bounded output, nonzero exit and descendant markers in WFL tooling suites; official-image run `35513765769` on `58362064` produced 41 functional passes and only the deliberate WFL assertion failure, exit 1; the temporary suite is removed | +| Keep Linux/Windows governance and Blacksmith nightly | Updated Governance provisions current nightly assets on both platforms; WFL job freshly resolves/pins Docker digest and records runtime/source revisions | +| Independent review and repaired findings | `orm-independent-review.md`, `independent-migration-review.md`, `runtime-review.md`, migration safety regressions and HTTP evidence distinguish source review from Maintainer approval; `type-analysis-review.md` records remaining runtime static-analysis diagnostics honestly | +| Final ready-to-merge five-section PR | PR #16 retains the five required sections, independent reviews, regression evidence and final exact-head checks; its Validation section identifies the final clean-run result, freshly resolved image digest and all source revisions | + +The user approved the initial three upstream merges and official nightly +publication. Those PRs are merged and official nightly 26.9.14 is published. +Linux verification exposed the runtime's 300-second whole-command cap; a fourth +upstream change adds an explicit finite invocation budget. Its review and all +exact-head CI passed, including real 305-second assertions on Linux and Windows. +The user approved its merge and replacement nightly publication; #741 is merged +as `3720dd74c82f4a1354aa64cfe66260ec3eccba93`. Official nightly 26.9.16 at +`23c1a4577da68d853fa30c49a17773427471eca4` has published successfully. The isolated +Scriptorium Red proof is recorded in `orm-verification.md`; final clean-head +acceptance is recorded in PR #16. +Scriptorium merging and production deployment remain outside this deliverable. diff --git a/docs/orm-implementation-evidence.md b/docs/orm-implementation-evidence.md new file mode 100644 index 0000000..78da33e --- /dev/null +++ b/docs/orm-implementation-evidence.md @@ -0,0 +1,109 @@ +# ORM implementation evidence + +This records implementation and local regression evidence. Application +integration, migration recovery, review fixes and the WFL-only runner are +implemented. [The verification record](orm-verification.md) tracks official +runtime publication and final Scriptorium CI separately. + +## Implemented library contracts + +The reusable library under `lib/orm` has no application imports. Its linear +include chain implements declarative fields/models/indexes/relationships, +record presence and validation, safe assignment, composed SQL predicates, +bounded ordered queries, owned/borrowed connections, CRUD/upserts and explicit +batched relationship loading. The public operations raise actionable errors +through the upstream `raise_error` prerequisite. They use native WFL database +handles, parameter binding and transaction scopes. + +`identity` fields represent SQLite integer identities as canonical signed +64-bit decimal text. They are opaque identifiers, not floating-point quantities. +Projection uses SQL `CAST` before WFL sees the value; comparisons bind the exact +text to the stored integer column. Sorting qualifies that stored column so its +numeric ordering survives the text projection. `integer` arithmetic fields use +the exact WFL numeric range; `number`, `text` and `boolean` are distinct types. +Null, an omitted field, an empty string and a missing record remain distinct. + +Queries default to 100 records, allow pages of 1–1000, and add the primary key as +a deterministic tie-breaker. Counts and existence describe the filter, ignoring +pagination. Bulk writes honor the requested page and ordering and reject a +structurally unrestricted filter without explicit `orm_whole_table` intent. +Null equality means `IS NULL`, null inequality means `IS NOT NULL`, and Boolean +groups retain SQLite's three-valued logic. Membership explicitly includes null +when requested. Text pattern searches escape SQL wildcard characters. + +Relationship access never issues implicit SQL. Loading batches at most 100 +distinct keys per query and pages related results in groups of 1000; a complete +page requires a following query to establish exhaustion. Loading is limited to +1000 source and 10000 related records; larger collections use explicit child +queries. Assembly currently compares collected rows to source keys in memory; +this is transparent bounded work, not a query per source row. Empty and orphan +relationships preserve their absence rather than deleting or rejecting rows. + +## Local WFL evidence + +All scenarios, assertions and fixture generation below are WFL. They ran on +Windows against the upstream transaction candidate reporting 26.9.12. The +candidate is an implementation build, not the published nightly. Relevant +capability and provenance evidence remains in `runtime-capabilities.md`. + +| Suite in `TestPrograms` | Latest result | Important boundary | +|---|---:|---| +| `orm-types.test.wfl` | 6 passed | Type/null/default checks, malicious identifiers, signed 64-bit identity bounds | +| `orm-models.test.wfl` | 5 passed | Duplicate identifiers, indexes, relationship uniqueness and model isolation | +| `orm-records.test.wfl` | 5 passed | Missing/null/empty, atomic assignment, sensitive fields, redacted errors | +| `orm-predicates.test.wfl` | 7 passed | File-backed injection, Boolean/null/membership semantics and literal wildcard searches | +| `orm-crud.test.wfl` | 10 passed | Defaults, exact generated IDs, projections, upserts, paging, guarded writes, native and nested rollback, connection reuse and ownership | +| `orm-query-order.test.wfl` | 1 passed | Numeric ordering with exact-text identity projection | +| `orm-relationships.test.wfl` | 3 passed | Two SQL queries for 101 parent keys, explicit loading, null/orphan results and no lazy queries | + +| `orm-relationship-limits.test.wfl` | 3 passed | 1001 children take two queries; 10001 fail without partial state; 1001 parents fail before SQL | +| `db-contracts.test.wfl` | 5 passed | Legacy result behavior, exact large IDs and atomic installer rollback | +| `db-review-contracts.test.wfl` | 8 passed | Independent NULL lookup, mutation metadata and nullable-key page regressions | + +The executable progression in `examples/orm/progression.test.wfl` passes three +cases using explicit migrations from the first saved record through composed +queries, transactions, relationship loading and reversible index changes. +The final combined candidate is `df6ad9a2`, executable SHA256 +`b96c06f6013a9a64d4d042c9b379a30af1c8d274d7855b5ebe9d7fe14a750d1c`. +All 16 existing database cases, three authentication cases and six rendering +cases pass. The final complete gate passed 41/41 suites, including the HTTP +review cases, legacy upgrade, migration recovery and pinned Scribe, on source +`3bc7b4f`. [The verification record](orm-verification.md) retains the exact local +provenance, verified official publication and the separate remote CI evidence. + +### Retained regression chronology + +- `052ba8d` records the new field contract before implementation. Its initial + run failed because the library did not exist; this is feature absence, not a + claimed behavioral regression. +- `afb4f9d` preserves a real isolation failure: adding a relationship to one + model also changed another model through a shared container list default + (expected 0, actual 1). Copy-on-write updates pass all five model assertions. +- `3f2ad08` preserves a real ordering failure: SQLite sorted the exact-text + projection alias, yielding identity 10 before 2. Qualifying the underlying + table column passes the unchanged order assertions. +- Record error assertions fail against the official runtime because the + required application error primitive is absent; they pass with the candidate. +- `ffeebbb` preserves large-ID rounding (expected exact text, actual rounded + Number) and partial installer state (expected zero users, actual one) before + replacing the application data layer; both now pass. +- `c0d0603` preserves independent review's seven failures plus a passing NULL + session guard. Physical row identity, explicit trusted updates and legacy + NULL-equality adaptation pass the unchanged eight assertions. + +### Timing and fixture isolation + +The original 60-second whole-file budget expired after five CRUD cases with a +debug build and seven with a release build. A timestamped release probe isolated +roughly 10.2 seconds in the two SQLite table-reset statements on this Windows +drive; the following insert and lookup took approximately 26 ms combined. +Durability settings were not relaxed. `TestPrograms/.wflcfg` now gives complete +database suites 180 seconds. Lock, HTTP and subprocess tests retain independent, +shorter deadline assertions, and the new runner must own a hard child timeout. + +WFL test blocks isolate parent variables by copying mutable values on parent +lookup. Mutable fixture objects such as an `OrmSession` therefore belong inside +each test. Relationship tests borrow the fixture's native handle into a local +session and assert changes on that same local instance. An independent source +inspection confirmed these test-environment semantics; they are not a lost +mutation bug in ordinary action calls. No query-count assertion was weakened. diff --git a/docs/orm-independent-review.md b/docs/orm-independent-review.md new file mode 100644 index 0000000..6e32c35 --- /dev/null +++ b/docs/orm-independent-review.md @@ -0,0 +1,79 @@ +# Independent ORM and adapter review + +Technical review on 2026-09-20; this is not Maintainer approval. + +## Preserved Red evidence + +`TestPrograms/db-review-contracts.test.wfl` ran with the combined WFL runtime +`ae5395d9bd215d0d9fa1c039e666f03c8897cf88` before the fixes. It reported **1 passed, +7 failed** (exit 1). The retained local log is +`target/review-probes/compatibility-promoted-red.log`. + +The first six cases use only existing application wrapper APIs. Those same +cases passed **6/6** against application baseline +`4ec5c88d9e4ae27041599ad8293a28130b56fbf2` using the same runtime, with only the +include path changed. Log: +`target/review-probes/compatibility-promoted-baseline.log`. +The last two cases exercise the new ORM's promised read/write page equivalence; +there is no predecessor ORM API to test at the baseline. + +Confirmed findings: + +- Nullable author equality changed legacy `column = ?` with NULL from matching + no rows into `IS NULL`, returning orphan posts and pages. +- Session insertion synthesized `last_insert_id = 0`, losing the native SQLite + rowid metadata of a text-primary-key insert. +- Sensitive user updates synthesized zero insert metadata instead of preserving + the native connection's most recent insert result. +- Bulk selection using a nullable key inside `IN (SELECT key ...)` omitted NULL + keys. Expired NULL-ID sessions were not purged. Paged updates and deletes of + ordinary SQLite tables also skipped the selected NULL-key row. + +A guard case verifies `session_delete(conn, nothing)` remains a no-op. It passes +before the fixes and must continue to pass when bulk operations gain physical +row identity. The bulk cases contain two separate NULL-key rows and one named +key; their postconditions ensure the selected row changes without modifying or +removing its neighbors. + +Expected remedies are confined to ordinary SQLite tables: retain physical row +identity for bounded mutation, preserve compatibility equality semantics in the +application wrappers, and return actual operation/connection metadata through +the ORM rather than synthesizing it. The Green and remedy review below records +the implemented fixes and their successful regression results. + +## Reviewed areas without findings + +The review covered model and field validation, identifier quoting, bound values, +predicate/operator validation, identity precision, defaults and missing-vs-NULL +mapping, sensitive assignment, safe bulk-write intent, ownership and errors, +relationship batching, and legacy result aliases/projections. No SQL injection +path was found in the ordinary value API. The explicit raw SQL escape hatch is +intentionally trusted application code. HTTP compatibility is separately +recorded in `tests/integration/EVIDENCE.md`. + +## Green and remedy review + +The owner's remedies pass all eight review cases on the same combined runtime; +retained log `target/capability-probes/db-review-green.log`. NULL compatibility +lookups now compile to a false predicate, while explicit ORM NULL equality still +means `IS NULL`. Nullable-key pages use a stable physical SQLite row identity +for read ordering and bounded writes. Insert results retain exact text rowid +metadata directly from `RETURNING`; sensitive updates use the native write result. + +The additional `TestPrograms/orm-write-review.test.wfl` cases passed **2/2** on +that runtime (`target/review-probes/rowid-sensitive.log` before promotion): + +- A table with declared `_ROWID_`, `rowid`, and `_orm_insert_rowid` fields retains + all logical values, exposes the actual physical rowid, and changes only the + selected second NULL-key row through the remaining `oid` alias. +- The trusted sensitive-update API rejects unrestricted queries, primary-key + changes and invalid field types before mutation. A SQL-looking text value is + bound literally without changing a neighboring sensitive field. + +The reviewed remedy addresses the confirmed findings. Physical-row selection +is scoped to ordinary SQLite tables with model declarations describing columns +that shadow rowid aliases; WITHOUT ROWID tables are outside the documented ORM +scope. New aliases are quoted and validated, and physical identity stays inside +SQL for mutation so large identities do not pass through floating-point values. +Final combined-suite and remote CI acceptance are separate from this technical +source review and focused evidence. diff --git a/docs/orm-migrations-design.md b/docs/orm-migrations-design.md new file mode 100644 index 0000000..9e5cfad --- /dev/null +++ b/docs/orm-migrations-design.md @@ -0,0 +1,128 @@ +# ORM and versioned migrations + +Status: implemented architecture with independent technical review and local +recovery verification, 2026-09-20. [The verification record](orm-verification.md) +tracks official runtime publication and final remote acceptance. +The actual API is documented in [orm.md](orm.md); administrative operations are +documented in [migrations.md](migrations.md). + +## Foundations and compatibility + +Read `G:\repos\wfl\Docs\wfl-foundation.md` in full. The local source revision +`cb1dadaad96939a4450a6eb2b3a6a51678035b7f` is also the source identified by the +2026-09-20 nightly release. Its foundations document includes the overriding +No-Unlearning Invariant, matching the quotation in the task attachment. The +only supplied attachment is the task text; it contains no separate copy of the +foundations, so a byte comparison with a second attachment is unavailable. +The local reference is the acceptance source specified by the task. + +One API must serve first examples and advanced applications. Model definitions, +explicit immutable migrations, typed records, parameterized queries, and scoped +transactions are present from the first example. No automatic schema sync, +destructive convenience defaults, or replacement expert query language. Use WFL +actions and containers with readable names, ordinary WFL values, and the +standard database/filesystem/HTTP/process facilities. Unsupported syntax and +deliberately triggered unrelated runtime errors are not acceptable API forms. + +The change is R3 under [testing.md](../testing.md). Preserve all existing +tables, data, IDs, sequence high-water marks, aliases, ordering, nullable values, +settings upserts, authentication and authorization behavior, extension hooks, +uploads, URLs, `data_dir`, and the 8080 port fallback. Legacy relationships do +not have foreign-key constraints or cascades; migration must not invent them. +Keep the Scribe gitlink and the existing application/include layout. + +## Implemented architecture + +The existing linear include chain now connects `db.wfl` through +`persistence-helpers.wfl` and `models.wfl` to `app/migrations.wfl`. That +bridge includes independent branches for `util.wfl`, the reusable ORM chain, +and immutable historical version definitions. Application model definitions +belong in `app/`; the ORM does not know Scriptorium's seven tables. +Existing `db.wfl` action names remain compatibility adapters. Explicit SQL +exceptions are limited to specialized SQLite time expressions or similarly +justified operations, use the same connection scope, and bind all values. + +Models describe table/field names, types, primary keys, server defaults, +nullability, uniqueness, indexes, sensitive assignments and relationships. +Identifiers must be checked against model definitions and quoted. A record +distinguishes an omitted field from explicit null and empty text. Omission +selects a declared default; null is accepted only for nullable fields. Reads +return mapped records; no matching record is `nothing`. Unknown or protected +mass-assignment fields fail before SQL. Database constraints remain authoritative +for concurrent writes. Diagnostics name the operation/model and corrective +action without logging parameter values or credentials. + +Queries compose predicates as an expression tree with explicit all/any groups, +null predicates, stable ordering, bounds, projection, count and existence. +Writes without a limiting predicate require explicit whole-table intent. +Relationships load explicitly; collection loads batch foreign keys so query +counts grow by bounded batches rather than one query per record. SQL text, +parameter count and query count are inspectable without exposing values. + +The owner opens/closes each database handle. Scoped transactions must route all +ORM and escape-hatch operations to the same underlying connection, propagate +validation/database errors, and roll back reliably. Nested behavior must be +explicit and tested; rejecting nesting before mutation is an acceptable +initial behavior. No simulated transactions or swallowed errors. + +## Migration lifecycle + +A WFL administrative entry point resolves the same `.wflcfg` data directory as +startup and prints the target path before mutation. Commands scaffold immutable +ordered migrations, inspect status, plan without mutation, migrate all/up to a +target, and roll back a count/to a target only after checking reversibility. +Historical definitions and their checksums cannot depend on current models. + +Runtime probes established that dynamic include aliases are unsupported. +Versions therefore use explicit static includes and an ordered typed registry. +Scaffolding creates a version file and gives exact registration instructions; +it does not pretend to discover dynamic exports. Checksums combine immutable +source (CRLF normalized to LF) with a deterministic ordered descriptor. + +The ledger records identifiers, names, checksums, timestamps and ordered apply/ +rollback events. Validate duplicate/order/history errors before schema writes. +Acquire SQLite serialization with a bounded lock policy. Each migration and its +ledger event share one real transaction. Interrupted work must leave neither +partial schema/data nor a successful event. Preflight irreversible rollback +before changing any migration in the requested range. + +Fresh databases begin with a migration. Unversioned adoption compares managed +table columns, constraints and indexes against supported historical shapes, +including sessions without `csrf_token`; never blindly stamp a baseline. +Inspect only managed objects for drift, preserving extension-owned tables, +indexes and triggers. Reject unknown/ambiguous managed schemas with recovery +instructions. Startup applies supported additive upgrades, never rollback. + +SQLite rebuilds must preserve actual rows, IDs, sequence state, constraints, +indexes, triggers and relationships; inspect foreign keys deliberately. A +discarded column's values cannot be recreated by adding an empty column. +Destructive changes require an irreversible declaration and forward repair or +a verified database-plus-uploads restore procedure. + +## Implementation and evidence sequence + +1. Inventory persistence and every existing test/driver. Verify source and + executable nightly capabilities before committing to API syntax. Keep minimal + WFL reproducers for any required upstream remedy. +2. Implement model/record/query/transaction primitives with WFL file-backed + regression tests and executable progressive examples. +3. Implement versioned migration operations, legacy adoption, drift checks, + rebuilds, locking and recovery tests before changing startup ownership. +4. Route every normal application persistence operation through the ORM and + preserve wrapper contracts. Exercise real HTTP installation, authentication, + publication, user/settings/media and throttling boundaries. +5. Replace Python tests and runner only after mapping their actual behavior to + WFL replacements. WFL owns fixtures, assertions, discovery, subprocesses, + HTTP requests, failure/timeout handling and cleanup. Shell/YAML only provision + and invoke WFL. Preserve the non-test hygiene checker and pinned Scribe suite. +6. Run focused and full suites, intentional failure propagation, hygiene, + backup/restore and adversarial recovery checks. Record exact runtime digest, + versions, revisions and final Blacksmith/Governance results. +7. Obtain independent technical review of foundation alignment, SQL safety, + transactions, adoption and recovery; fix findings and rerun affected checks. + Audit every task requirement, update actual shipped documentation, then open + the completed five-section PR. Maintainer approval/merge remains separate. + +Any confirmed runtime blocker remains visible and prevents a production-ready +claim. Do not replace missing WFL test capabilities with another language or +mark this plan as the completed deliverable. diff --git a/docs/orm-verification.md b/docs/orm-verification.md new file mode 100644 index 0000000..9769ba0 --- /dev/null +++ b/docs/orm-verification.md @@ -0,0 +1,280 @@ +# ORM verification record + +This record distinguishes local candidates, verified official publication and +the complete runner's isolated remote failure proof. The final clean-run result, +exact PR head and GitHub merge checkout are recorded in the Validation section +of [Scriptorium PR #16](https://github.com/WebFirstLanguage/Scriptorium/pull/16). +Local candidate passes are not substitutes for that final official-image gate. + +## Runtime prerequisites + +The replacement official nightly +[35512196018](https://github.com/WebFirstLanguage/wfl/actions/runs/35512196018) +completed successfully at source `23c1a4577da68d853fa30c49a17773427471eca4`, +version **26.9.16**. It contains all four reviewed and approved upstream changes, +including #741's explicit finite invocation budget. Its version-only commit is +the direct child of #741's merge `3720dd74c82f4a1354aa64cfe66260ec3eccba93`; +tag `v26.9.16` resolves to that same source. + +The published image is +`bsbyrdwfl/wfl@sha256:1092a0c557731113f08673c764e0deb9e0b053992b4488974863f0a8d692db91`. +Independent inspection of the Docker publication job verified version, source, +both current tags and all five consumer checks, including assertion-failure +exit 1. Both native packaging jobs and the complete release gates passed. +Each platform's integration job passed 164 WFL programs, 36 documentation +checks and three web checks, including the real 305-second assertion. + +The exact governance provisioning script verified the fresh canonical manifest, +immutable Windows MSI sidecar and downloaded bytes, extracted without installing, +and confirmed version 26.9.16. MSI SHA256: +`73db4e8b9f725b6e1953ba453ca972e659a0388e0d343d4afea3b13fb8cfddec`. +Extracted executable SHA256: +`32375058e0111f6c159144a8ffa9bd5b3a3bde81f578eca015149053ea4c6cbb`. +The immutable Linux archive SHA256 is +`8cd9c852afe069f689fee014c71a9667202be9e92344f139f1605b5b167c3fe6`. +Both Governance job summaries independently record these platform artifacts, +version and source. The complete Scriptorium command now runs against this +official runtime; the isolated remote deliberate-failure proof follows below. + +### Initial publication history + +The initial three separate upstream changes have independent technical review, +successful exact-head remote CI and approved merges. Official nightly +[35506079498](https://github.com/WebFirstLanguage/wfl/actions/runs/35506079498) +completed successfully for their combined source +`8d82d785ea59300834de1c48b04a7ba0e187a1cd`, version `26.9.14`. + +Published Docker digest: +`bsbyrdwfl/wfl@sha256:8498abec67995274cb4d6cc2ee9580be11f70e15af20497bb62e51d4b2058e3c`. +The immutable Windows MSI SHA256 is +`a3ff0b5c8dd2141536004298583fb698c20596fb5a49ad90d53f56d4ac15e3d4`; +the Linux archive SHA256 is +`b6b74d37ad4baae4bf7333c8a27f5dcc332104cdf7257c6da6754ea1fc7c809d`. +The exact governance provisioning script downloaded, verified and extracted +the Windows MSI locally and confirmed its version. Both publication jobs and +all nightly validation gates passed. The same-day GitHub mirror remains at +26.9.12 by its immutable-release policy; governance uses the canonical CDN. +A retained one-hour CDN cache entry initially returned the old manifest even +with `Cache-Control: no-cache`. A unique request key returned the verified new +manifest and is now part of governance provisioning. Immutable asset downloads +and checksums are still independently verified. + +| Upstream PR | Tested head | Remote CI | +| --- | --- | --- | +| [HTTP redirect and repeated-header controls #737](https://github.com/WebFirstLanguage/wfl/pull/737) | `b8e5e8768a13c44aa033a3cd732b85f089262eff` | [35500755237](https://github.com/WebFirstLanguage/wfl/actions/runs/35500755237), Linux 169 and Windows 168 WFL programs passed | +| [Owned process lifecycle and diagnostics #738](https://github.com/WebFirstLanguage/wfl/pull/738) | `f54243f43769b8d6e8ebe0b8f54ed2b2a735b234` | [35505056152](https://github.com/WebFirstLanguage/wfl/actions/runs/35505056152), Linux 174 and Windows 173 WFL programs passed | +| [Application errors and pinned schema transactions #739](https://github.com/WebFirstLanguage/wfl/pull/739) | `59709aed97d045dbd9a04093fff5c32ae60a95e3` | [35505568741](https://github.com/WebFirstLanguage/wfl/actions/runs/35505568741), Linux 181 and Windows 180 WFL programs passed | + +The merge commits are `eecd658c32e23abdb6f4e0cc881d8af26dacd2d2` (#737), +`d6993e77578892e0a93d3b8c8b08d6b1b55eede8` (#738), and +`8d82d785ea59300834de1c48b04a7ba0e187a1cd` (#739). The final schema PR's +combined Linux/Windows integration gates each passed 157 programs, 36 +documentation checks and three web checks. Its Rust gate passed 2429 tests +with 27 existing ignored tests. + +The listed WFL gates had zero failures and zero timeouts. Existing platform +skips remain recorded by upstream CI; these counts do not claim every upstream +scenario ran on every platform. Workspace tests, formatting, strict Clippy, +database integration, documentation and web checks also passed in those runs. + +The combined local candidate is WFL source +`df6ad9a2f9f401d943edfdec2e3804941a9c513f`, reporting version `26.9.12`. +Its Windows executable SHA256 is +`b96c06f6013a9a64d4d042c9b379a30af1c8d274d7855b5ebe9d7fe14a750d1c`. +Scribe remains pinned to `93d62af5a6ed6c3ce257ef888107fc3ca1e2dc1d`. + +## Local complete-suite gate + +The final candidate complete runner passed **41/41 discovered suites**, zero +failures, exit 0, on Scriptorium source +`3bc7b4faaf9c000047d940f0e420067b777db5c9`. This includes the historical-index +review regression, strengthened CLI CRLF fixture and nested HTTP probe cleanup. +The exact command was `target/runtime/combined-df6ad9a2/wfl.exe scripts/run_tests.wfl`. +The ignored local log is `target/full-candidate-final.log`. The earlier complete +40-suite result is retained in `target/full-candidate-df6ad9a2-green.log`. + +`python scripts/check_repo_hygiene.py` passed with 179 candidate paths; Git +reported a clean worktree and `git diff --check` passed. The source audit found +no active Python test implementation or driver. The only non-WFL executable code +under tracked tests/scripts is the hygiene checker implementation and the existing +Scribe update utility. + +The hygiene checker fixtures create disposable Git repositories and need child +Git execution permission. A first sandboxed complete run passed 39/40 suites; +the tooling suite received an OS access denial starting Git. With child-process +access, the unchanged assertions passed. This environment failure is not +represented as a product test pass or hidden by skipping the fixture cases. + +Runtime analysis can print nonfatal type diagnostics while executing these +tests. [The independent analysis report](type-analysis-review.md) distinguishes +reproduced analyzer limitations from intentional unknown-type safety checks. +This record does not claim a clean static-analysis gate. + +## Official-image CI acceptance + +Commit `b9e7903286495d6665b7cfdbd8916a4dc5abd952` adds a temporary WFL assertion +with marker `SCRIPTORIUM_INTENTIONAL_CI_FAILURE_PROOF`. On the combined local +runtime, `scripts/run_tests.wfl --group application` reports 24 suites, +23 passed, one failed, exit 1; only that deliberate suite fails. The retained +log is `target/ci-propagation-red-local.log`. Its required remote assertion-failure +proof has now completed, and the deliberate test is removed from the final +source. Normal complete-suite discovery contains 41 suites. + +[Blacksmith run 35513765769](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35513765769), +[job 106086113517](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35513765769/job/106086113517), +tested PR head `58362064ac9594c0a7e35d66dd968e72220bc285` through merge checkout +`b3d92ba400ad55a49535e8393c1964bc3338904f` against base +`4ec5c88d9e4ae27041599ad8293a28130b56fbf2`. It freshly pulled and resolved the +official 26.9.16 image `sha256:1092a0c557731113f08673c764e0deb9e0b053992b4488974863f0a8d692db91`, +recorded WFL source `23c1a4577da68d853fa30c49a17773427471eca4` and unchanged +Scribe `93d62af5a6ed6c3ce257ef888107fc3ca1e2dc1d`, then executed +`wfl --execution-timeout 1200 scripts/run_tests.wfl`. + +The result was **42 suites: 41 passed, one failed**, with runner and Actions +exit 1. The sole top-level failure was `TestPrograms/ci-propagation.test.wfl`, +showing the named marker and the exact deliberate assertion. Every functional +suite completed successfully, including all eight integrations, tooling, +runtime probes, executable progression and pinned Scribe. The runner's nested +negative fixtures are intentional assertions about failure propagation; their +containing suite passed. Container cleanup passed. Independent log review +confirmed no additional failure or parent deadline expiry. Suite execution ran +from 13:32:02.4093265 to 13:36:25.8055402 UTC (about 263 seconds); the separate +upstream 305-second tests establish the longer-budget boundary. + +Both [Governance platforms](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35513765906) +passed three suites and 41 assertions each (28 hygiene, four migration CLI, +nine runner), plus static hygiene with 180 paths including the deliberate file. +All six [runtime/media gates](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35513764169) +passed 15 assertions and cleanup. Their runtime identities agree with the +official publication. After removing only the deliberate suite, the required +clean gate runs the same full command with 41 suites; final-head results and +provenance are retained in the PR's Validation section. + +### Earlier official-image attempts and runtime review + +The first full run on the published 26.9.14 image, +[35507634634](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35507634634), +tested PR head `07e8adcb7785c168c37fd56cc35e88492809a820` through merge checkout +`86185103700e28cd12ba9379f00adebdd056dfbd`. It reported **32 passed, 10 failed**, +exit 1: the intentional assertion, an oversized-upload transport failure, and +eight failures after the runner reached the runtime's 300-second whole-command +budget. This is not isolated failure-propagation proof. All five standalone +[runtime capability gates](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35507632213) +passed. Both [Governance platforms](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35507634656) +passed hygiene 28, migration CLI 4 and runner 9 cases (three suites), followed +by repository hygiene with 180 paths including the deliberate test. + +The same published Windows runtime completed **42 suites: 41 passed, one +intentional failure**, exit 1, in approximately 160 seconds. Its executable +SHA256 is `da109d5926f6af4a2f24c45150140764c406c055aef2dd7e43e45b85278f1dfe`; +the log is `target/full-official-windows-red.log`. Independent timing comparison +found migration CLI tests took 35.1 seconds on native Linux, 36.1 in Docker and +12.6 on Windows. No connection leak or fixed delay was found. The longer full +Linux run needs an explicit finite invocation budget while retaining child +deadlines. The media fixture now uses a portable rejection boundary while still +requiring HTTP 413, no stored rows or files, and a subsequent successful request; +see `tests/integration/EVIDENCE.md`. + +The next complete run, +[35508311002](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35508311002), +tested head `147b15c88c5fbd29e4826a32a0db5daf747ae59d` through merge checkout +`1c190c5a1798ca86244b9bb0142251f35c78b3b6` on the same official runtime. +Media passed **2/2** both within this complete run and in its new focused Linux +gate. The complete runner still reported **32 passed, 10 failed**, exit 1: +the intentional assertion and nine failures after the 300-second parent deadline. +Both Governance platforms and all six focused capability jobs passed. This is +evidence that the media correction works, not isolated deliberate-failure proof. + +A fourth upstream change, +[WFL #741](https://github.com/WebFirstLanguage/wfl/pull/741) at +`358dc9eb15f61152f75f816d9e51c396cd24dd77`, has passed independent review and all +exact-head CI checks. It adds an explicit finite +`--execution-timeout` option for the shared invocation budget. The prepared +complete-suite command is `wfl --execution-timeout 1200 scripts/run_tests.wfl`. +It preserves existing per-suite timeouts and child process limits. Official +26.9.16 now contains this option; the isolated official-image Red is recorded +above and final clean-head acceptance is recorded in the PR. +Independent technical review found no remaining source or fixture blocker; +25 fast WFL cases, a real 305-second WFL boundary assertion, 2414 existing Rust +tests (27 existing ignored), formatting, strict Clippy and 36 documentation checks passed +locally. Exact-head upstream CI +[35510552984](https://github.com/WebFirstLanguage/wfl/actions/runs/35510552984), +Docker validation, configuration lint and CodeQL all passed. Both integration +jobs passed 164 WFL programs, 36 documentation checks and three web checks; +24 existing integration skips remain recorded. Program sweeps passed 188 on +Linux and 187 on Windows, with zero failures or timeouts. All seven new CLI +budget suites ran on both platforms. Independent log inspection verified the +real long-boundary assertion passed after 305.009 seconds on Linux and 305.161 +seconds on Windows. These jobs tested merge checkout +`23a523bfcfe3d5015b1ab3da6dbb3272512f769f` containing the exact PR head. +The user approved this additional merge and publication. PR #741 is merged as +`3720dd74c82f4a1354aa64cfe66260ec3eccba93`; the replacement official nightly +publication is verified above. + +The prepared command completed locally with the reviewed CLI candidate: +**42 suites, 41 passed, one intentional failure**, exit 1. Only +`TestPrograms/ci-propagation.test.wfl` failed; every functional suite completed. +The candidate reports WFL 26.9.15, with executable SHA256 +`1185f150c8214d982a27431d4b88b94f7f20d6ce6c718161c35120deb817650f`. +It was built from upstream Red `84cb272c` plus the reviewed, frozen CLI source +change before its Green commit. Scriptorium was at +`2d1d6e2a5ca1ea98396f1d587218b7279144077f` plus the prepared command/help and +documentation changes; test scenarios were unchanged. The log +`target/full-cli-budget-candidate-red.log` spans 11:45:41–11:54:10 UTC on +2026-09-20, approximately 509 seconds by file metadata. Repository hygiene passed +with 180 paths including the intentional suite. This proves local consumer +behavior beyond 300 seconds, not a final official-image CI pass. + +Subsequent review of #741 found that a shorter override could alter server HTTP +timeouts, a final duration wait could miss expiry, and dump modes could ignore +a misplaced option. The reviewed remedy preserves the original per-operation +duration, checks sleeping/receiving waits without dropping active WFL handlers, +and rejects the misplaced option before output. New WFL regressions cover +buffered and streamed HTTP, final waits, owned-child cleanup observed before any +test-side process reap, WebSocket listener release, and main-loop exemption. +All 25 focused WFL cases and the fresh 305-second boundary check passed locally. + +The remedy at WFL `30ed9462` was retested against clean Scriptorium source +`df8039cc252e2e48772ed88b9d273b98e30a67c5` with the same complete command: +**42 suites, 41 functional passes, one deliberate failure**, exit 1. The WFL +26.9.15 candidate executable SHA256 is +`c0619c544b09551ce7988f58a564f50ff04e48a5e94a6f903c08a141b911c516`. +Log `target/full-cli-budget-reviewed-red.log` spans 12:16:55–12:20:17 UTC, +approximately 202 seconds. This latter consumer run does not itself prove the +300-second boundary; the separately repeated long WFL test does. + +The first #741 CI run, +[35509162373](https://github.com/WebFirstLanguage/wfl/actions/runs/35509162373), +failed in an unchanged Windows trusted-proxy fixture when its previously probed +port was occupied before the WFL server bound it. Linux integration was canceled +by matrix fail-fast; neither long-duration step ran. The final fixture now binds +port zero and discovers the actual owned address, retaining all existing +assertions; its seven cases passed locally and in the successful final Windows +integration job. The replacement official publication also passed; no failed +or canceled job is counted as a pass. + +The implementation and final-revision check record are in +[Scriptorium PR #16](https://github.com/WebFirstLanguage/Scriptorium/pull/16). +Its first complete-suite attempt, +[35504493284](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35504493284), +successfully pulled the official image and provisioned the disposable container, +then failed before test execution because that image lacks the new process CWD +syntax. The log also reports `current_executable` as undefined. This is the +expected unmet runtime prerequisite, not an accepted failing test or the +deliberate assertion-failure proof. + +That attempt used PR head `26bc1c2eb9e19a604bf91c9b43171e433ff56101`, GitHub's +tested merge checkout `4545f6246dbdaac704aa0f18c19b76e30a0073bd`, WFL `26.9.12`, +Scribe `93d62af5a6ed6c3ce257ef888107fc3ca1e2dc1d`, and image +`bsbyrdwfl/wfl@sha256:7ddc51e6320affa7cfe26263fece590ddbdebe5582659b7e660ca823ed3ddbf1`. +Container cleanup succeeded. Both Linux and Windows +[Governance jobs](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35504493283) +also provisioned the official nightly successfully and failed on the same +missing process syntax; their later hygiene steps were skipped, not passed. + +The initial runtime-capability Red in +[35499009581](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35499009581) +used Scriptorium `a81ecf9c2c68784f3deace8b3521feae762e51ff` and image +`bsbyrdwfl/wfl@sha256:7ddc51e6320affa7cfe26263fece590ddbdebe5582659b7e660ca823ed3ddbf1`. +It proves the original runtime gaps; it is not a substitute for deliberate +failure-propagation proof through the completed new test runner. diff --git a/docs/orm.md b/docs/orm.md new file mode 100644 index 0000000..a90b39d --- /dev/null +++ b/docs/orm.md @@ -0,0 +1,212 @@ +# Models, records and queries + +The SQLite ORM lives in `lib/orm/` and has no application imports. Include +`lib/orm/migrations.wfl` for the complete API, or a lower module when schema +management is not needed. Includes form one chain. Scriptorium connects that +chain to its existing `util → db → auth → render` tree in +`app/migrations.wfl`; Scribe remains an unchanged upstream dependency. + +Run the executable progression from the repository root: + +```text +wfl --test examples/orm/progression.test.wfl +wfl scripts/run_tests.wfl --group examples +``` + +The first example applies explicit versions before saving a record. The next +uses the same records and queries inside a native transaction, then loads a +relationship explicitly. The final example plans, reverses and reapplies an +index migration without losing rows. Current models and immutable historical +definitions are separate files. There is no automatic schema synchronization. + +## Declare a model + +```wfl +create new OrmField as identity_field: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes +end +create new OrmField as title_field: + field_name is "title" +end +create new OrmModel as notes: + table_name is "notes" + fields is [identity_field and title_field] +end +``` + +This declares a mapping; a versioned migration creates its physical table. +Models require exactly one primary key. Composite primary keys are unsupported. +Fields declare `nullable`, `unique_value`, `sensitive`, `has_default` and +`default_value`. A text field can use `server_default is "current timestamp"` +for SQLite's UTC `datetime('now')`. Declare ordinary or unique indexes using +`OrmIndex` (`index_name`, `field_names`, `unique_values`) in the model's +`indexes` list. Identifiers must contain ASCII letters, digits or underscores +and start with a letter or underscore. They are quoted before use in SQL. + +| Type | WFL value | Stored SQLite affinity | +| --- | --- | --- | +| `text` (default) | Text, including empty text | TEXT | +| `integer` | Whole Number within ±9,007,199,254,740,991 | INTEGER | +| `number` | Finite Number | REAL | +| `boolean` | `yes` or `no` | INTEGER, mapped from 1 or 0 | +| `identity` | Canonical signed 64-bit decimal Text | INTEGER | + +Identities use text because WFL Numbers cannot represent every SQLite integer. +For example, `"9007199254740993"` remains exact. There is no numeric coercion: +`"1"` is an identity, `1` is an integer, and `yes` is a boolean. Stored +values are validated when mapped; invalid existing data produces a repair +diagnostic rather than a lossy conversion. Database uniqueness and other +constraints remain authoritative under concurrent writers. + +## Save and find records + +```wfl +store draft as orm_record of notes +call orm_set with draft and "title" and "Publish the release" +store saved as orm_save of session and draft +store loaded as orm_find_key of session and notes and (orm_get of saved and "id") +``` + +`orm_save` returns the mapped saved record, including database-generated values. +Keep that return value when editing an existing record. A new record remains +new if the caller discards it. Saving a persisted record updates its assigned +non-key fields; a deleted record or changed primary key raises an error. + +| Situation | Behavior | +| --- | --- | +| Unassigned field | `orm_has` is false; `orm_get` raises an actionable error | +| Explicit `nothing` | Stored as NULL only if the field is nullable | +| Empty text | An ordinary text value, distinct from missing and NULL | +| Missing field with a default | Omitted on INSERT; the database supplies the default | +| Missing required field | Rejected before INSERT | +| No matching row | `orm_first` and `orm_find_key` return `nothing` | +| Empty collection | `orm_find` returns an empty list | +| Projected-out field | Missing, never silently represented as NULL | + +`orm_assign(record, assignments)` takes explicit +`orm_value(field_name, field_value)` assignments. It validates the entire batch +before changing the record. Unknown fields, repeated fields, invalid types and +sensitive fields are rejected. Use `orm_set_sensitive` only after the +application has authorized that particular password, role or token change. +It is an explicit trusted operation, not a form-data mass-assignment path. + +`orm_insert_if_absent(session, record, conflict_field)` ignores only the +declared unique/key conflict. Other constraint failures still raise. +`orm_upsert` updates explicitly assigned non-key fields on that conflict. +Neither performs a race-prone existence check followed by an unprotected insert. + +## Compose queries + +Start with `orm_query(model)`; modify it with `orm_where`, `orm_select`, +`orm_order_by` and `orm_page`, then use `orm_find`, `orm_first`, +`orm_count` or `orm_exists`. The predicate constructor is +`orm_compare(field_name, comparison, value)`. Combine predicates with +`orm_all(list)`, `orm_any(list)` and `orm_not(predicate)`. + +Comparisons: `equal`, `not equal`, `less than`, `less or equal`, +`greater than`, `greater or equal`, `in`, `is null`, `is not null`, +`contains`, and `starts with`. NULL equality compiles to IS NULL, and NULL +inequality to IS NOT NULL. Other comparisons use SQL's three-valued logic: +unknown results do not match, including after NOT. IN lists can contain NULL +explicitly and hold at most 500 values; empty IN matches nothing. Empty ALL +matches everything; empty ANY matches nothing. Text search escapes literal +`%`, `_` and escape characters before binding the pattern. Case behavior +is SQLite's default LIKE behavior; the ORM does not promise Unicode folding. + +Order directions are exactly `ascending` and `descending`. The primary key +is appended to ordering for stable pages. Stored integer identities sort +numerically even though the returned identity is text. Pages default to 100, +allow 1–1000 rows, and require a nonnegative safe integer offset. Counts and +existence checks describe the filter and ignore the page window. + +`orm_update(session, query, assignments)` and `orm_delete(session, query)` +honor the query's bounded page and ordering. They reject a known unrestricted +predicate unless `orm_whole_table(query)` explicitly records that intent. +Bulk updates reject primary-key changes and unchecked sensitive assignments. +The explicit trusted `orm_update_sensitive(session, query, field, value)` +counterpart authorizes one field while retaining the same selection guards. +For a large operation, use an explicit transaction and deliberate batches; +avoid offset pagination when deleting rows that shift subsequent offsets. + +## Load relationships explicitly + +`OrmRelationship` declares `relation_name`, `related_model`, +`local_field`, `related_field` and `many`. Add it with +`model.relate(relationship)`. Types must agree; a scalar relationship requires +a unique target field. Logical relationships do not add physical foreign keys, +cascades or cleanup. Migration schema declarations own those decisions. + +Call `orm_load(session, records, relation_name)`, then +`orm_related(record, relation_name)`. Reading a relationship never issues SQL. +An unloaded relationship raises, an absent scalar returns `nothing`, and +an absent collection returns an empty list. Orphan references remain intact. + +Loading accepts at most 1000 parents, groups up to 100 distinct keys per query, +and reads related rows in pages of 1000. The total eager result is limited to +10,000 related records. Exceeding the limit raises before any source record is +marked loaded; query the related model in explicit pages for larger results. +101 distinct parent keys with a small result take two queries. 1001 children +of one parent take two queries. An exact multiple of 1000 requires an empty +final page to establish completion. Matching uses in-memory scans across the +bounded parent/result sets, so cost grows with both sizes. + +## Connections, transactions and diagnostics + +`orm_open(path)` owns a SQLite connection. Close it in a `finally` block. +`orm_borrow(conn)` wraps an application-owned native handle; closing the ORM +session does not close that handle. Closing a session is idempotent, and later +operations on that session fail. + +```wfl +store conn as session.connection +in transaction on conn: + store saved as orm_save of session and draft +end transaction +``` + +Use the same native transaction form at every level. It pins the connection, +commits on normal completion (including a normal return), and rolls back on an +error. A false return value is normal completion, not a rollback request. +Raise an error inside the block; catch it outside. Nested transactions on the +same connection are rejected and no savepoints are implied. Migration schema +transactions add the explicit `for schema changes` clause and perform foreign +key validation before commit. See [migrations](migrations.md). + +`session.query_count`, `last_sql` and `last_parameter_count` expose query +cost and generated SQL without logging bound values. SQL errors retain useful +database diagnostics with assigned text redacted; validation errors identify +the operation and corrective action without printing rejected values. +`orm_sql_query(session, authored_sql, bound_values)` and +`orm_sql_execute` are the clearly marked escape hatch. Bind all runtime +values; identifier and SQL-expression safety remains the author's responsibility. +Native WFL rejects raw transaction-control SQL. + +The supported schema uses ordinary SQLite rowid tables. Models must declare +all stored columns that shadow a rowid alias. Nullable text primary +keys use a nonshadowed physical rowid to distinguish and order NULL-key rows +in bulk pages. Leave at least one of `_rowid_`, `rowid`, or `oid` undeclared. +INSERT records expose the exact physical identity as `inserted_rowid`, separate +from the logical primary key; this preserves the legacy session insert result. + +## Application compatibility and foundations + +`app/db.wfl` retains the application's map-returning actions. Normal data +access for users, sessions, posts, pages, settings, media and login attempts +uses ORM models and queries. The explicit SQL exceptions in +`app/persistence-helpers.wfl` read SQLite's clock and year; historical timestamp +semantics remain unchanged. The adapters preserve aliases, ordering, empty +settings, nullable fields and orphan references. Ordinary IDs remain Numbers; +IDs outside the safe numeric range are exact text through lookups and routes. +The installer now commits its user and settings atomically. + +The No-Unlearning Invariant is implemented by one model/record/query API, +explicit migrations from the first runnable example, and the same native +transaction form throughout. Advanced use adds predicates, projections and +relationships to existing queries. Validation is explicit and errors suggest +repair. Safe defaults include bound values, identifier validation, protected +fields, bounded reads and guarded bulk writes. Query counters, documented +batching and hard eager limits make performance visible. SQLite-specific +schema behavior is documented rather than hidden behind speculative backends. diff --git a/docs/persistence-inventory.md b/docs/persistence-inventory.md new file mode 100644 index 0000000..eca66c2 --- /dev/null +++ b/docs/persistence-inventory.md @@ -0,0 +1,315 @@ +# Persistence compatibility inventory + +This is the pre-ORM compatibility baseline at Scriptorium revision +`4ec5c88d9e4ae27041599ad8293a28130b56fbf2`, inspected on 2026-09-20. It records +actual source behavior, including distinctions omitted by the older architecture +summary. It is an input to implementation and acceptance testing, not a claim +that the requested ORM or migration lifecycle has shipped. + +## Sources and ownership + +The complete application include path is +`main → site_ext → render → auth → db → util`; `render` also includes the pinned +Scribe root. Preserve this tree, wrapper signatures, and the raw database handle +passed to `site_ext_boot` and `site_ext_dispatch`. Scribe stays upstream-owned. + +`app/db.wfl` owns seven application tables and one explicit index. Most data +access is there, but `auth.session_start` directly queries SQLite for seven-day +expiry, and `render.site_context` directly queries SQLite for the year. +`main.wfl` has no direct SQL. Extensions may own other tables, indexes, triggers, +and SQL through the same connection; core schema inspection must not reject, +rewrite, drop, or adopt those objects. + +## Exact legacy schema + +The baseline creates these statements with `IF NOT EXISTS`. No table has foreign +keys, cascade behavior, enum checks, custom collation, generated columns, or a +`STRICT`/`WITHOUT ROWID` declaration. `role` and `status` values shown in UI docs +are application conventions, not database constraints. + +```sql +CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'author', created_at TEXT DEFAULT (datetime('now'))); +CREATE TABLE sessions (id TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL, csrf_token TEXT NOT NULL DEFAULT ''); +CREATE TABLE posts (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT UNIQUE NOT NULL, title TEXT NOT NULL, body_markdown TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'draft', author_id INTEGER, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now'))); +CREATE TABLE pages (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT UNIQUE NOT NULL, title TEXT NOT NULL, body_markdown TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'draft', author_id INTEGER, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now'))); +CREATE TABLE settings (skey TEXT PRIMARY KEY, svalue TEXT NOT NULL DEFAULT ''); +CREATE TABLE media (id INTEGER PRIMARY KEY AUTOINCREMENT, filename TEXT UNIQUE NOT NULL, original_name TEXT NOT NULL DEFAULT '', content_type TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0, uploader_id INTEGER, created_at TEXT DEFAULT (datetime('now'))); +CREATE TABLE login_attempts (id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, attempted_at TEXT DEFAULT (datetime('now'))); +CREATE INDEX idx_login_attempts_ip_time ON login_attempts (ip, attempted_at); +``` + +Five integer primary keys use `AUTOINCREMENT`: users, posts, pages, media, and +login_attempts. A rebuild must preserve `sqlite_sequence`, including a high-water +mark above the current maximum row ID. The implicit unique indexes for username, +slugs, filename, and text primary keys are significant even though the app names +only the rate-limit index. + +All timestamp columns and the author/uploader references are nullable. The text +primary keys have no explicit `NOT NULL` clause. Do not strengthen those legacy +constraints during adoption. Existing orphans and nulls are valid legacy state; +deleting a user does not delete content, media, sessions, or attempts. + +The older supported sessions definition is identical except it lacks +`csrf_token`. The old upgrader checks +`pragma_table_info('sessions')` and adds +`csrf_token TEXT NOT NULL DEFAULT ''` only when missing. Existing session IDs, +users, timestamps, and expirations are untouched. Its tests also construct a +database containing only this old sessions table; all remaining tables are +then created by `db_migrate`. + +## Value and result contracts + +- Single-row wrappers return a row map or WFL `nothing` when no row matches. + Collection wrappers return a list, including an empty list. Counts return the + numeric `n` value of `count(*)`; no count wrapper returns the SQL row map. +- SQL NULL maps to `nothing`; empty text remains empty text. `first_row` tests + list length, not row truthiness. Defaults apply when INSERT omits a field; + they must not silently replace explicit NULL or empty text. +- User, session, post, page, media, and attempt mutation wrappers return the + native `execute` result, not an inserted ID or an ORM record. Current WFL + source defines it as a map with `affected_rows` and `last_insert_id`. Keep that + wrapper contract even if the reusable ORM offers record-returning methods. +- `setting_get` returns its fallback only when the key has no row. An existing + empty setting is returned unchanged. `setting_set`, `setting_default`, + `install_mark_done`, and successful `db_migrate` return `yes`. +- `field_or` treats missing keys and `nothing` as absent, but preserves empty + text, `no`, and zero. `map_has` also treats a present `nothing` as absent; it + is therefore unsuitable for ORM missing-versus-NULL validation. +- Query projections below are contracts. In particular, list rows intentionally + omit password hashes and several content fields; changing every result to a + full model record would change those contracts. + +## Wrapper inventory + +All predicates containing caller values use bound parameters. Quoted constants +such as `published`, time expressions, and sort keys are static source SQL. +No wrapper interpolates caller text into an identifier or SQL fragment. + +### Settings and installer + +| Action | Behavior | Application callers | +|---|---|---| +| `setting_get(conn, key, fallback)` | Select `svalue` by `skey`; fallback only for no row | `site_context`, home pagination, installer form, settings form, `install_is_done` | +| `setting_set(conn, key, value)` | `INSERT … ON CONFLICT(skey) DO UPDATE SET svalue = ?`; returns yes | Settings save, installation title/tagline, installed flag | +| `setting_default(conn, key, value)` | `INSERT OR IGNORE`; existing value wins | Boot defaults | +| `install_is_done(conn)` | True only for setting `installed` exactly `yes` | Dispatch gate, installer POST guard, boot backfill and startup message | +| `install_mark_done(conn)` | Upsert `installed=yes` | Installation and boot backfill | +| `install_apply(conn, title, tagline, username, hash)` | Create admin, then set title/tagline/installed; catches user-create failure and returns no | Installer POST | + +The old `install_apply` is not atomic. Its promised compatibility is that an +initial username failure leaves settings untouched; an error after user creation +can leave a partially installed site, whose next boot locks the installer because +a user exists. The new transaction implementation should make installation +atomic while retaining successful results and the duplicate-username no result. +Do not weaken the boot backfill for pre-wizard installations. + +### Users + +| Action | Result or mutation | Application callers | +|---|---|---| +| `user_count(conn)` | Numeric count of all users | Boot backfill, dashboard | +| `user_create(conn, username, hash, role)` | Insert three named values; DB generates ID/time | Installer, admin user create | +| `user_by_username(conn, username)` | `id, username, password_hash, role, created_at` | Login verification, installer session creation | +| `user_by_id(conn, id)` | Same projection | Admin edit/update | +| `user_list(conn)` | `id, username, role, created_at`; `ORDER BY id ASC` | Admin users list | +| `user_set_role(conn, id, role)` | Update only role by ID | Admin user update | +| `user_set_password(conn, id, hash)` | Update only hash by ID | Admin user update when password nonempty | +| `user_delete(conn, id)` | Delete only user row by ID | Admin delete after self-delete guard | + +Passwords are already hashes when passed to persistence; never hash them again +during adoption or mapping. `role` and `password_hash` need explicit trusted +assignment paths, while the generic ORM must reject unchecked mass assignment. +Username update is not offered by the existing wrapper. User deletion leaves +references intact; surviving sessions cease resolving because their user join +no longer matches. + +### Sessions and login attempts + +| Action | Result or predicate | Application callers | +|---|---|---| +| `session_create(conn, sid, user_id, expires_at, csrf)` | Insert supplied session ID/user/expiry/token; SQLite generates created time | `auth.session_start` | +| `session_user(conn, sid)` | Inner join users; `u.id AS id, u.username AS username, u.role AS role, s.csrf_token AS csrf_token`; `s.id=? AND expires_at > datetime('now')` | `current_user`, request dispatch | +| `session_delete(conn, sid)` | Delete only matching session | Logout | +| `session_purge_expired(conn)` | Delete where expiry `<= datetime('now')` | Boot | +| `login_attempt_record(conn, ip)` | Insert IP; SQLite generates ID/time | Invalid credentials and invalid/failed installer inputs | +| `login_attempts_recent(conn, ip)` | Count matching IP with `attempted_at > datetime('now', '-15 minutes')` | Login and installer POST | +| `login_attempts_clear(conn, ip)` | Delete all attempts for that IP | Successful login/install | +| `login_attempts_purge(conn)` | Delete attempts `<= datetime('now', '-15 minutes')` | Boot | + +`session_start` generates a 32-byte random hex ID and a separate CSRF token, +obtains expiry with `SELECT datetime('now', '+7 days') AS exp`, then calls +`session_create`. Preserve SQLite UTC text values and strict time boundaries. +Legacy sessions gaining an empty CSRF token remain identifiable; `csrf_ok` +rejects an empty token and does not invent one during lookup or upgrade. + +The HTTP limiter refuses requests once the recent count is **at least 10**, +before credentials or CSRF are checked. Login CSRF failure does not add an +attempt; bad credentials do. Installer field-validation and apply failures add +attempts; successful authentication clears only the requesting IP. + +### Posts + +`post_create` inserts slug, title, Markdown body, status, and author ID. +`post_update` updates slug/title/body/status and sets `updated_at=datetime('now')` +for one ID; it preserves author and creation time. `post_delete` deletes one ID. +Their callers are the corresponding create/update/delete HTTP handlers, which +enforce CSRF and author/admin ownership outside the wrapper. + +| Read action | Exact projection | Filter and order | +|---|---|---| +| `post_by_id` | `id, slug, title, body_markdown, status, author_id, created_at, updated_at, author_name` | ID; left join username as author_name | +| `post_by_slug` | Same | Slug only, including drafts; left join username | +| `post_list_published` | `id, slug, title, body_markdown, status, created_at, author_name` | Published; created_at DESC, id DESC; bound LIMIT/OFFSET | +| `post_list_all` | `id, slug, title, status, author_id, updated_at, author_name` | updated_at DESC, id DESC | +| `post_list_by_author` | Same list projection | author_id; updated_at DESC, id DESC | +| `post_count_published` | Numeric count | Published only | +| `posts_total` | Numeric count | All rows | + +The author join is LEFT JOIN in every post read, so null and dangling authors +retain the post with `author_name=nothing`. The public post handler separately +rejects drafts; moving its filter into `post_by_slug` would change that public +wrapper. Public home uses published count/list and `posts_per_page`, falling +back to 5 if parsed value is below 1. Admin dashboard/list calls all/by-author +according to role. Edit/update/delete first call `post_by_id` for existence and +ownership. Duplicate slug creation is caught by the handler; other database +constraints remain authoritative. + +### Pages + +Page create/update/delete mirror posts, including updated time and preserving +author/created time. There is no user join in any page wrapper. + +| Read action | Exact projection | Filter and order | +|---|---|---| +| `page_by_id` | `id, slug, title, body_markdown, status, author_id, updated_at` | ID | +| `page_by_slug` | Same | Slug AND published | +| `page_list_all` | `id, slug, title, status, updated_at` | title ASC | +| `page_list_by_author` | Same list projection | author_id; title ASC | +| `page_list_published` | `id, slug, title` | Published; title ASC | +| `pages_total` | Numeric count | All rows | + +Page ordering has no explicit secondary key. Do not silently promise or impose +an ID tie-breaker as a compatibility assumption. Public rendering uses the slug +lookup; `site_context` builds navigation with the published list; admin list +chooses all/by-author; edit/update/delete use ID lookup and ownership checks. + +### Media + +`media_create` inserts filename, original_name, content_type, size, uploader_id; +SQLite creates ID/time. `media_by_id` returns +`id, filename, original_name, content_type, size, uploader_id, created_at` or +nothing. `media_list_all` returns those fields plus left-joined username as +`uploader_name`, ordered `created_at DESC, id DESC`. `media_delete` removes only +the row by ID. Null/dangling uploaders do not remove a media list row. + +The upload handler writes bytes first, then inserts metadata; an insertion +failure attempts to delete the just-written file. Delete checks uploader/admin +ownership, deletes the row, then attempts file deletion. ORM migration must not +introduce cascades or rewrite filenames. Preserve `/assets/uploads/` URLs and +the storage path chosen at boot. Database transactions alone cannot make a +filesystem write/delete atomic. + +## Startup and extension contract + +Current boot performs these operations in order: + +1. Read optional `.wflcfg` from process CWD. `config_value_from` trims lines, + ignores whole-line `#` comments, uses the first exact key, and retains inline + `#` text. Missing config reads as empty text. +2. Resolve HTTP port through `config_port_from`: numeric integer 1–65535 or + fallback 8080. Resolve `data_dir`; create it if nonempty and absent. Defaults + are `scriptorium.db` and `static/uploads`; configured paths are + `/scriptorium.db` and `/uploads`. +3. Configure the public theme and issue existing missing/refused-theme notices. +4. Open `sqlite://` plus resolved database path and run `db_migrate`. +5. Insert only absent settings: site_title=Scriptorium, + site_tagline=Words, well kept., posts_per_page=5. +6. If any user exists and installed is not exactly yes, upsert installed=yes. +7. Call `site_ext_boot(db)` once with the open database handle. +8. Purge expired sessions and old attempts; create uploads directory if absent. +9. Listen on the resolved port; print matching startup/installer URLs; dispatch + requests using the same open connection until shutdown. + +Administrative migration commands must resolve the same path and visibly name +it. Read-only status/plan must not accidentally initialize an absent database. +Startup must stop before serving if migration/adoption fails. It may perform +supported forward upgrades; it must never roll back destructively. Preserve the +extension's place after successful schema/default/backfill and before listening. + +At request dispatch, assets are served before the installation lock. An +uninstalled site sends every other route to `/install`; an installed site's +late installer GET or POST redirects without applying setup. Public extension +dispatch precedes stock public routes, returns yes only after responding, and +otherwise falls through. Its signature is +`site_ext_dispatch(db, req, req_method, req_path, req_body, user)`. + +## Adoption and ORM acceptance decisions + +These concrete requirements follow from the inventory: + +1. Validate the core table structure, defaults, uniqueness, primary keys, and + owned index before recording any baseline. Recognize fresh, exact current, + and supported pre-CSRF schemas explicitly. Recognize safe partial legacy + creation only when each present managed object matches its historical + definition. Stop on incompatible or ambiguous objects without a successful + ledger row. Preserve every extension-owned object and row. +2. Use immutable migration-specific definitions, independent of current model + metadata. Adoption is an inspected history event, never permission to claim + that unexecuted arbitrary migrations ran. Preserve passwords, IDs, nullable + values, defaults, sessions, CSRF tokens, timestamps, and sequence state. +3. Keep existing wrappers as projection/return adapters over one reusable ORM. + Model metadata can describe logical relationships without adding physical + foreign keys to legacy tables. Explicit relationship loading must preserve + nullable/dangling links and avoid one-query-per-row collection loading. +4. Preserve existing native connection ownership for callers and extensions. + ORM-owned connections need explicit close semantics; borrowed handles must + not be closed unexpectedly. Transaction scope must use verified native WFL + behavior and documented nested semantics, never catch-and-report-success. +5. Parameterize every value, validate metadata identifiers and sort directions, + distinguish missing/null/empty/default/not-found, reject unknown fields, and + require explicit authority for sensitive assignment and unfiltered bulk + mutation. Keep counts and generated SQL inspectable without parameter values. +6. Keep the seven-day expiry, fifteen-minute attempt window, setting upsert, + SQLite timestamp defaults/updates, and year lookup as explicit, named uses + of a parameterized SQL escape hatch if the generic query API cannot express + them naturally. Exceptions must not become a second persistence layer. +7. Characterize these exact wrapper projections, ordering, empty results, + return maps, duplicate errors, null/orphan relationships, installation lock, + and SQL time boundaries before refactoring. Exercise file-backed adoption, + restart, and HTTP callers afterward; existing memory tests alone are not + recovery evidence. + +The WFL foundations were read in full from +`G:\repos\wfl\Docs\wfl-foundation.md`. The supplied task attachment includes the +same No-Unlearning Invariant: “For every feature, the beginner form and the expert +form must be the same form, or connected by a smooth path with nothing to +unlearn.” No separate foundations attachment was present in the supplied +references, so there is no second full text to compare. The implementation must +use explicit versioned migrations in the first executable example and extend +the same model/record/query vocabulary through filters, relationships, +transactions, and schema evolution. Avoid opaque encoded metadata strings, +disposable automatic synchronization, and silently swallowed errors. Readable +WFL actions, strict validation, actionable safe diagnostics, native standard +library integration, and bounded/query-visible relationship loading make the +foundations measurable acceptance conditions. + +## Documentation corrections identified + +- `docs/ARCHITECTURE.md` constraint 2 says main includes render directly; actual + main includes site_ext. Its diagram already shows the correct tree. +- Constraint 4 says csrf ALTER is guarded by try; source now inspects schema + and propagates ALTER errors. Its no-transactions claim must be rechecked + against the required current nightly, not copied as a runtime limitation. +- The database-layer description and README layout say all queries live in + db.wfl; auth expiry and render year queries are counterexamples. +- Architecture's theme extension recipe mentions an active-theme setting; + current implementation selects `theme`/`theme_root` from `.wflcfg`. +- Architecture and README prose say more than 10 failures; actual HTTP + threshold is at least 10 recent failures. +- `main.wfl` header and `auth.sid_from_cookie` comment claim request values + only resolve in the main loop, although architecture already records that + limitation as lifted and media upload reads request values in an action. +- Existing docs correctly describe no general migration lifecycle at the + baseline. README, CLAUDE, SECURITY, testing, CONTRIBUTING, hygiene policy, + and workflow commands must describe actual shipped WFL migration/testing + behavior when integration is complete. diff --git a/docs/runtime-capabilities.md b/docs/runtime-capabilities.md new file mode 100644 index 0000000..984cd4e --- /dev/null +++ b/docs/runtime-capabilities.md @@ -0,0 +1,231 @@ +# ORM and migration runtime capability audit + +This historical audit records the runtime gaps found before implementing the +ORM and its upstream prerequisites. All executable probes are WFL. See +[the verification record](orm-verification.md) for the implemented remedies, +merged upstream revisions and final acceptance status. + +## Provenance and commands + +Verified on Windows on 2026-09-20 with the official Windows nightly +**WFL 26.9.12**, extracted locally without replacing the installed 26.9.11: + +```powershell +$wfl = 'G:\repos\Scriptorium\target\runtime\nightly-26.9.12\PFiles\wfl\bin\wfl.exe' +& $wfl --version +& $wfl --test tests/runtime/orm-native-baseline.test.wfl +& $wfl --test tests/runtime/transaction-validation-capability.test.wfl +& $wfl --test tests/runtime/rebuild-foreign-keys-capability.test.wfl +``` + +Downloaded MSI SHA-256: `34422e832d267b5add2e8380a2992a2a2f0756f2ed6984af20f724a590cff956`. +Extracted executable SHA-256: `4e9c8f63d02bf84497fdf23d9c20efda3a6c609b2925a2a4a94ee4348bd85afa`. + +The source inspected is WFL commit +`cb1dadaad96939a4450a6eb2b3a6a51678035b7f`. The installed 26.9.11 also passed the +upstream native transaction and include-diamond suites, but it is not treated +as current-nightly evidence. Remote Docker provenance and results must be +recorded separately; these Windows results do not establish Linux behavior. + +Portable commands, with the selected WFL on `PATH`: + +```sh +wfl --test tests/runtime/orm-native-baseline.test.wfl +wfl --test tests/runtime/transaction-validation-capability.test.wfl +wfl --test tests/runtime/rebuild-foreign-keys-capability.test.wfl +``` + +These suites derive the checkout location from `script_directory`, use +synthetic file-backed databases under ignored `target/runtime-capability-results`, +and close and delete their databases in `finally`. They do not open site data. +Each capability suite has its own database name. Do not run the same suite +concurrently against the same checkout. + +| Probe | Observed result on 26.9.12 | +|---|---| +| `orm-native-baseline.test.wfl` | 6 passed, exit 0; positive capabilities and explicitly labeled existing limitations | +| `transaction-validation-capability.test.wfl` | 1 failed, exit 1: expected zero retained rows, observed one | +| `rebuild-foreign-keys-capability.test.wfl` | 1 failed, exit 1: expected one extension row, observed zero | +| Upstream `TestPrograms/database_transaction_test.wfl` | 8 passed: commit, native-error rollback, read-own-writes, transaction SQL rejection, nested-block rejection | +| Upstream `TestPrograms/modules/include_diamond.wfl` | 2 passed | +| Upstream `TestPrograms/containers/documented_features.wfl` | 14 passed | + +The failing probes assert the required safety outcome, rather than relaxing an +assertion to make the currently unsafe alternative green. Neither failure is +a parser error. They do not assert that the runtime promises Boolean-return +rollback or safe arbitrary table rebuilds today. + +## Remote nightly evidence + +[Blacksmith run 35499009581](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35499009581) +tested Scriptorium `a81ecf9c2c68784f3deace8b3521feae762e51ff` and pinned Scribe +`93d62af5a6ed6c3ce257ef888107fc3ca1e2dc1d`. The job freshly pulled the nightly, +resolved and ran `bsbyrdwfl/wfl@sha256:7ddc51e6320affa7cfe26263fece590ddbdebe5582659b7e660ca823ed3ddbf1`, +which reported **WFL 26.9.12**. Each suite ran in WFL in a disposable container; +no Python test implementation or runner was used by this capability workflow. + +The native baseline passed 6/6. Transaction validation failed 1/1 with a +retained row; the rebuild failed 1/1 with its extension row deleted; HTTP failed +1/1 with final status 200 and no intermediate cookie; process control passed +1/4 and failed cwd isolation, post-wait output, and stderr preservation. All +failed assertions produced exit 1 and failed their CI jobs. All container cleanup +steps passed. This is prerequisite Red evidence, not successful validation of +the requested ORM or complete converted test suite. + +The user subsequently authorized upstream WFL fixes and separate prerequisite +PRs, followed by completion of Scriptorium. Work is isolated into transaction, +HTTP response control, and process lifecycle branches. Existing Scriptorium +application code and Python checks remain unchanged until their replacements +can meet the full contracts. The existing five WFL application suites also +passed locally on 26.9.12 (60 tests); the runtime emitted its existing included- +action/type-analysis warnings. These local baseline results are not final PR CI. + +## Capabilities available now + +- Typed containers, mutable instance properties through actions, inheritance, + interfaces, defaults, and lists of instances work in the upstream container + suite. A small ORM can represent model, record, query and relationship + state with containers. +- Maps can be declared, indexed and checked with `record contains "field"`. + Presence distinguishes an absent field from an explicitly present `nothing`. + Empty text remains a different value. Missing map indexing raises an error. + A map's `for each` loop yields values, not keys. Direct indexed mutation + (`change record["field"] to value`) is not supported by the assignment + grammar. There is no registered map-key enumeration/mutation builtin. + This is an API-design constraint, not by itself a proof that an ORM is + impossible: a record container can encapsulate an ordered field collection. +- WFL `nothing` and SQL NULL compare as no value. Parsed JSON null also compares + equal to `nothing`, but `typeof` reports `Nothing` for JSON null and `Null` + for the literal/SQL value. Use `isnothing`, not a single `typeof` spelling, + for the ORM's semantic null check. +- SQLite values are bound separately from SQL through `and parameters [...]`. + An injection-shaped text value and a bound `nothing` round-trip unchanged. + Write results contain `affected_rows` and `last_insert_id`. +- `sqlite_schema`, `pragma_table_info(?)`, and other SQLite introspection + queries are available through normal parameterized queries. Migration + adoption must inspect those results instead of assuming a baseline. +- Native `in transaction on conn:` pins one connection for its body. Database + errors roll back preceding DDL and ledger writes together and preserve the + original diagnostic through `when error` and `error_message`. Commits and + rollbacks report failures instead of silently claiming success. +- Native nested transaction blocks on the same handle are explicitly rejected; + raw BEGIN, COMMIT, ROLLBACK, SAVEPOINT and RELEASE through query/execute are + also rejected. A documented reject-nesting policy is feasible; savepoint + behavior must not be claimed. +- Include diamonds now work. The old architecture limitation is obsolete on + this nightly, as is its statement that transactions do not exist. This audit + does not authorize rearranging Scriptorium's existing include chain. +- SQLite integers become WFL floating-point numbers. The baseline demonstrates + that SQL `9007199254740993` reads and rebinds as `9007199254740992`. Exact + legacy IDs require an explicit text-preserving representation, including + casts for reads; silently coercing every ID to Number loses data. + +Relevant source at the inspected revision: + +- [SQL parameters, connection configuration and pooling](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/database.rs#L60) +- [Transaction SQL guard](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/database.rs#L211) +- [Native transaction control](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/mod.rs#L7092) +- [Integer decoding](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/database.rs#L423) +- [Assignment grammar](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/parser/stmt/variables.rs#L120) +- [Include execution in the parent scope](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/mod.rs#L8932) + +## Required capability: application errors that abort a transaction + +`save_then_reject` in the failing probe writes one row inside a native +transaction, then returns `no` to indicate a later validation failure. The +caller observes failure but the row is committed. That is the documented +native meaning of an ordinary return; changing all false returns to rollback +would be surprising and is **not** the proposed fix. + +The problem is that WFL does not expose a native statement or standard-library +operation that raises an application error, or rethrows a caught error after +adding safe operation context. Its AST/registered builtins have no such +operation. The illustrative `throw error` in an older upstream example is +not implemented: a separate exploratory execution reports undefined variable +`throw`. This parser/symbol observation is supporting evidence, not the Red +regression test. `expect` cannot substitute: it is restricted to test mode, +and it also records a test failure before throwing there. + +Returning an error object has the same ordinary-return problem. Catching a +native database exception and returning a failure value likewise consumes the +error that the transaction must see. Causing an unrelated division, parse or +SQL failure would obscure the actual cause and teach an implementation trick +instead of a coherent transaction/error API. Stopping the whole program does +not provide recoverable library error propagation. + +Upstream should provide one natural-language application-error operation, +usable in ordinary actions and containers, with these semantics: + +1. Raise a catchable runtime error with an application-supplied safe message; + preserve the cause/diagnostic when propagating a caught error. +2. Unwind `finally` blocks and native transaction scopes, rolling back before + the error reaches the caller. +3. Keep ordinary successful returns, including Boolean values, unchanged. +4. Surface uncaught errors through the CLI as nonzero failure. +5. Test nested calls, caught versus uncaught errors, rollback failures, and + errors occurring after both schema and ledger writes on file-backed SQLite. + +This is required for the requested composable validation/transaction API; +prevalidating one standalone save does not cover an arbitrary transaction +whose later operation fails application validation. + +Source: [transaction return/rollback branches](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/mod.rs#L7140), +[expect's test-mode restriction](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/mod.rs#L13387), +and [error-handler parsing](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/parser/stmt/errors.rs). + +## Required capability: connection setup before a migration transaction + +The rebuild probe creates a managed parent and an extension-owned child with +`ON DELETE CASCADE`. It attempts the usual create/copy/drop/rename rebuild in +a native transaction after requesting `foreign_keys = OFF` and +`defer_foreign_keys = ON`. SQLite reports `foreign_keys = 1`: changing that +setting inside an active transaction does nothing. Dropping the old parent +then deletes the extension's child row, even though an equivalent parent +exists at commit and the deferred foreign-key check can pass. + +This is an unsafe proposed migration strategy, not a claim that SQLite's +cascade semantics are defective. Existing extension relationships must be +preserved, so the migration system cannot simply assume they do not exist or +claim that deferred checks preserve the rows. + +The runtime creates a five-connection pool for file-backed databases. An +ordinary PRAGMA before the block can configure a different connection than +the one the block later pins. The public database API offers neither +connection options for this setting nor a session/checkout scope that pins a +connection before starting the transaction. The URL remainder is passed as a +filename, not parsed as SQLite connection options. Repeating a PRAGMA until it +appears to stick is not proof that the future transaction uses that connection. + +Upstream should expose a dedicated connection or connection-configuration +scope that guarantees setup and the subsequent transaction use the same +connection. A migration runner must be able to disable enforcement before +BEGIN, perform its rebuild and ledger update atomically, inspect +`foreign_key_check` and refuse commit on violations, then restore enforcement +and close its owned connection on every path. Per-connection options on a +separate migration pool could also provide the necessary guarantee if all its +connections receive those options. Normal application connections should keep +foreign-key enforcement enabled. + +The complete migration implementation would still need tests for preserved +indexes, triggers, sequences, extension objects and every supported legacy +schema. This probe establishes the missing primitive for the proposed rebuild +strategy; it does not substitute for those implementation tests. + +Source: [SQLite filename and pool construction](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/database.rs#L104) +and [transaction creation](https://github.com/WebFirstLanguage/wfl/blob/cb1dadaad96939a4450a6eb2b3a6a51678035b7f/src/interpreter/database.rs#L345). + +## Foundations acceptance + +The full local `Docs/wfl-foundation.md` (Version 2) was read. The task's pasted +attachment contains the requested implementation requirements and quotes the +same No-Unlearning Invariant; it is not a second complete foundation text. +No contradictory version of the invariant was supplied. + +The relevant acceptance criterion remains one API from the first model and +explicit migration through validation, composed queries and transactions. +The required upstream operations should fit that API without forcing callers +to replace Boolean-return examples with artificial parse failures, learn +connection-pool accidents, or accept silent data loss. These observations +block a production-ready claim; they do not justify a reduced feature set, +Python test fallback, or shipping an unintegrated prototype as the requested +finished ORM. diff --git a/docs/runtime-review.md b/docs/runtime-review.md new file mode 100644 index 0000000..efb3719 --- /dev/null +++ b/docs/runtime-review.md @@ -0,0 +1,252 @@ +# Independent runtime prerequisite review + +This historical review covers the initial runtime gaps and subsequent source +remedies. [The verification record](orm-verification.md) tracks their merges, +official publication and final application acceptance. + +Reviewed 2026-09-20 by a separate agent that did not author the capability +inventory or proposed ORM. This is technical review, not Maintainer approval +and not review of an implemented ORM. No ORM implementation exists in the +reviewed diff. The assessment concerns the complete requested production scope, +not whether a smaller demonstration can be written. + +Sources: WFL revision `cb1dadaad96939a4450a6eb2b3a6a51678035b7f` and the downloaded +Windows nightly executable +`target/runtime/nightly-26.9.12/PFiles/wfl/bin/wfl.exe`, reporting WFL 26.9.12. +The source revision was independently checked with Git. These are local Windows +results; they do not substitute for the required resolved Docker digest and +final Blacksmith runs. + +## Reviewed verdict + +| Capability | Verdict for the full requested scope | +|---|---| +| Custom abort/rethrow | Qualified blocker: preflight validation and contextual result APIs work now; application-directed abort after an earlier write in a general transaction scope does not | +| SQLite rebuild with extension foreign keys | Blocker for general safe transactional parent-table rebuild; additive migrations without rebuild are feasible now | +| HTTP redirect inspection | Blocker for honest real HTTP installation/login/redirect/cookie assertions through the current native client | +| Per-child working directory | Blocker for preserving the portable runner's existing directory/disposable-Scribe contracts without a new native primitive or an independently verified compliant launcher | +| Spawned child diagnostics/completion | Spawn/kill/poll/finally work; stderr and reliable final stream retrieval are missing for a complete timed runner with preserved diagnostics | + +Do not describe the runtime as lacking transactions, HTTP, subprocesses, or +error handling generally. Those facilities exist and several pass executable +probes. The missing behaviors below are specific. + +## Validation results versus aborting an active transaction + +`execute_transaction_statement` in `src/interpreter/mod.rs` commits normal +`return`, `break`, and `continue`. It rolls back runtime errors and abrupt +program exit. `database.rs` deliberately rejects SQL transaction-control +statements because ordinary operations use pooled connections. There is no +implemented user-defined throw/rethrow statement or explicit transaction abort. +The proposed `throw error "..."` probe reports `Undefined variable 'throw'`; +that is an unrelated failure, not supported error syntax. + +The absence of throw is **not** a reason to reject every explicit-result API. +I wrote and ran `target/review-probes/result-preflight.test.wfl`: **2/2 passed** +on a synthetic file-backed database. It proves both: + +1. A validation action can return an actionable failure map before a transaction + begins, preserving all rows and cleanly accepting a later valid insert. +2. A real unique-constraint error rolls back earlier writes; a catch outside the + transaction can preserve the database diagnostic and add safe operation and + corrective context. + +That is a foundation-aligned option for ordinary validation and a fixed batch +whose entire input can be checked in advance. An explicit result type can be +used consistently from the first example. It must not be represented as an +automatic rollback signal: the existing `orm-capabilities.test.wfl` probe +returns `no` after an insert and confirms that the row **commits**. + +For a general user-composed transaction, later validation can depend on reads +and earlier operations inside the transaction. Returning a failure at that +point commits earlier writes. Catching an error inside the scope and returning +a failure has the same issue. Prevalidating a fixed list outside the scope does +not solve that broader requirement or protect database-dependent checks from +races. Killing the application, deliberately referencing an undefined name, +dividing by zero, or issuing deliberately invalid SQL is not an acceptable +normal API for requesting rollback. + +**Required remedy for that broader scope:** a native, catchable application +error and rethrow facility, or a native scoped transaction abort with an +explicit failure result. It must run cleanup, preserve original diagnostics, +roll back the current transaction, propagate out of nested actions, and never +turn an aborted operation into success. Exact syntax is an upstream design +decision, not an assumed capability. Native rejection of nested transactions +before mutation is a valid initial documented nesting policy. + +## SQLite rebuild and extension-owned foreign keys + +Independently reran `target/capability-probes/rebuild-foreign-keys.wfl`: + +```text +foreign_keys before toggle in transaction: 1 +foreign_keys after toggle in transaction: 1 +extension rows after rebuild: 0 +``` + +The probe creates a managed parent and an extension-owned child with +`ON DELETE CASCADE`, then rebuilds the parent inside a native transaction. +Setting foreign keys OFF inside the transaction is ineffective; deferred +checking does not disable cascade actions. Dropping the old parent deletes the +child row despite replacing the parent with identical IDs. + +WFL `database.rs::connect` constructs a file-backed pool with up to five +connections. Every ordinary query/execute can acquire a different connection; +the transaction reserves its connection only when it begins. Therefore a +pre-transaction `PRAGMA foreign_keys=OFF` cannot establish the required +connection-local state reliably. The current URL handling takes the SQLite +suffix as a filename, not a supported connection-options API. + +SQLite's documented generalized rebuild procedure disables foreign-key +enforcement before beginning the transaction, rebuilds and checks the schema, +then restores enforcement after commit. Changing `foreign_keys` during an +active transaction has no effect. See the official +[ALTER TABLE procedure](https://www.sqlite.org/lang_altertable.html#making_other_kinds_of_table_schema_changes) +and [foreign_keys pragma](https://www.sqlite.org/pragma.html#pragma_foreign_keys). + +Refusing an unsafe rebuild with an actionable explanation is the correct +current behavior, but does not deliver the requested complete migration +capability. Copying back cascading child rows is not a general remedy: unknown +extension triggers, constraints, and cascading relationships may create further +effects. Reordering rename/drop steps can rewrite references to the temporary +table. Editing `sqlite_schema` or slipping transaction SQL past the guard is not +a production substitute for connection ownership. + +**Required remedy:** a native pinned connection/migration transaction facility +that configures foreign keys before BEGIN and restores the prior setting after +both success and failure; alternatively a documented dedicated connection +option applied to every connection plus verified transaction affinity. The +caller must be able to run foreign-key checks and abort before commit. Test +extension CASCADE/RESTRICT/SET NULL relationships, triggers, row preservation, +sequence high-water marks, rollback, interruption, and restoration of settings. +The ordinary ORM should continue enforcing foreign keys by default. + +## HTTP redirects and authentication cookies + +Independently reran `target/testing-probes/redirect.wfl`. The local child sends +a 302 with Location and Set-Cookie, then serves the redirect target. The WFL +caller receives only the final `status: 200`, `body: final response`; the initial +Location and synthetic session cookie are absent. + +`IoClient::http_client` builds a shared reqwest client with its default redirect +policy. `HttpRequestStatement` has method/headers/body options but no redirect +control. `send_http_request` exposes only the response returned by that client. +Response headers become a map of one text value per name, collapsing duplicate +names. Stream entry points also use the same client. + +A test that reads a new session ID directly from SQLite can check persistence, +but cannot prove that the login HTTP response delivered that cookie or the +expected redirect. Replacing the application's redirect response for tests is +also not the real application boundary. Raw socket clients or an external HTTP +driver are not available as an established WFL-only substitute here. + +**Required remedy:** per-request redirect control, retaining existing following +behavior by default, with a no-follow choice returning the actual 3xx status, +body, Location and Set-Cookie values. Preserve repeated response headers through +a list-valued accessor without changing existing single-value header accesses. +Keep request deadlines, cancellation, and response-size limits in force. A +cookie jar is optional: tests can manage cookies explicitly in WFL once the +actual response headers are observable. + +## Process isolation, output and cleanup + +Independent runs of existing probes produced: + +- `cwd.wfl`: the child launched by absolute path reports the repository CWD, + not its isolated fixture directory. Process AST/parser and command builders + have no working-directory argument; `wfl --help` and CLI parser expose none. +- `output.wfl`: foreground execution returns exit/stdout/stderr. A spawned + nonexistent WFL file yields exit 1 but its diagnostic cannot be obtained with + `read output from process`, which reads only stdout. +- `lifecycle.wfl`: output can be read before wait; after waiting, the process + handle is invalid. `wait_for_process` removes the handle and returns only + exit code, without joining the background collectors for stream EOF. +- Existing kill/finally probes and source show native kill, running-state poll, + and cleanup on ordinary errors/assertion failures. They are useful available + primitives, not blockers themselves. + +The existing runner explicitly guarantees repository CWD even when invoked +elsewhere, a separate disposable Scribe CWD/build tree, per-suite timeout, +continued execution after failure, and diagnostics. The Python runner tests +assert these behaviors. Merely supplying an absolute script path does not +preserve them. Running Scribe against the real checkout's build tree weakens +the isolation guarantee. + +Shell may provision or launch WFL under the user's task rules, but that does +not automatically establish a portable solution. A Unix launcher can replace +itself with WFL; a Windows `cmd` wrapper creates another process whose child is +not owned by WFL's process handle. Killing only that wrapper can leave the test +or server alive. Moving assertions, fixtures, or polling into shell would also +violate the task. No safe portable launcher was demonstrated in this review. + +Foreground execution already captures both streams and bounds execution. It is +valid for simple commands and prevents a blanket claim that stderr capture is +impossible. It lacks a per-command timeout/CWD option in the current language, +while the spawned path needed for polling lacks stderr and a final-output +completion result. Reading stdout, waiting, and hoping the collectors drained +is not deterministic; an arbitrary delay does not prove complete diagnostics. + +**Required remedy for the preserved portable runner:** per-command working +directory and timeout options with native process ownership, plus a completion +result containing exit code and fully drained bounded stdout/stderr. Preserve +existing syntax/results by adding options or a new explicit completion form. +Make cleanup deterministic and idempotent, including failures/timeouts, and +prove that no test/server child remains. A safely owned direct WFL child does +not require a general shell process-tree API; arbitrary descendants need an +explicitly documented ownership policy. + +## Review boundaries and next evidence + +The reviewed capability failures are not permission to weaken the requested +ORM, migration recovery, HTTP journeys, or WFL-only suite. Safe subfeatures can +be implemented while upstream prerequisites are addressed, but the full task +must not be declared complete on those subfeatures alone. + +Upstream fixes need WFL regression scenarios for each failed behavior, including +negative cases, then the full Scriptorium suite and real HTTP/recovery checks +against the resolved nightly image. Revisit this document after those fixes: +it records the reviewed runtime revision, not permanent WFL limitations. + +## Upstream remedy review, 2026-09-20 + +The baseline findings above remain evidence about `cb1dada`, not the updated +runtime candidates. The following reviews concern the prerequisite changes; +they do not constitute Maintainer approval or final Scriptorium acceptance. + +- HTTP: branch `codex/http-response-controls`, Green commit + `b8e5e8768a13c44aa033a3cd732b85f089262eff`, implements per-request + `and without following redirects` for buffered and streaming responses and + additive `header_values` arrays while retaining scalar `headers` and default + redirect following. A different agent independently reviewed this author's + change and found no blocking source issue. Its WFL regression suite passed + 8/8; the locked workspace suite passed 2,414 tests with 27 existing ignores; + the existing gated runner passed 145 WFL programs with 24 existing skips. + Formatting, strict Clippy, and fuzz-workspace compilation passed. Evidence + and existing environment skips are recorded in the upstream commit. +- Transactions: independent source review identified a cancellation lifetime + gap: a dropped transaction-body future could leave its transaction owned by + the registry while the interpreter remained alive. The owner reproduced it + with a real WFL concurrent HTTP peer and added an exact-slot + `TransactionBlockGuard` from reservation through body/commit. Re-review + confirmed that its synchronous registry removal releases the transaction on + cancellation, allowing the connection guard to roll back and restore foreign + keys. No remaining blocking source finding was identified; the owner's + rebuilt WFL cancellation regression and final gates remain separate evidence. +- Processes: independent source review found that applying a child working + directory after authorizing a relative explicit executable could change + which file executed. Both launch paths now freeze the explicit executable's + canonical parent-directory identity before applying child CWD. Review also + found a source-fixer mismatch for the merged contextual `with code name` + token; conservative operand protection fixes the rename mismatch. WFL + regressions cover exact-path authorization across CWD and fix-then-run exit + status. Re-review found both issues resolved and no remaining blocking source + finding; final rebuilt tests and platform gates are the owner's evidence. + +The ORM query-counter investigation did not establish another prerequisite. +WFL tests use an isolated environment, and each inherited mutable-value lookup +deep-clones the parent value. A module-scoped session therefore produces a new +container copy on another lookup; nested action argument binding itself retains +the instance. Tests should construct a local borrowed session for each case, +which retains real mutation and query-count assertions without sharing mutable +fixture state. The testing guide documents isolation generally; stable cached +copies of inherited mutable fixtures are not a documented contract. diff --git a/docs/type-analysis-review.md b/docs/type-analysis-review.md new file mode 100644 index 0000000..32dada6 --- /dev/null +++ b/docs/type-analysis-review.md @@ -0,0 +1,149 @@ +# Typed-container diagnostic review + +Technical investigation on 2026-09-20, without application-source changes, +annotation removal, diagnostic filtering, or upstream implementation changes. +This review does not represent Maintainer approval. + +## Runtime and probes + +Runtime: combined WFL `df6ad9a2f9f401d943edfdec2e3804941a9c513f`, reporting 26.9.12, +SHA-256 `b96c06f6013a9a64d4d042c9b379a30af1c8d274d7855b5ebe9d7fe14a750d1c`. +The eight minimal WFL probes and captured output remain under ignored +`target/type-analysis-probes/`. Each executable probe was run as: + +```text +target/runtime/combined-df6ad9a2/wfl.exe --test target/type-analysis-probes/.test.wfl +``` + +All eight execution tests passed, one assertion case per probe. These are +runtime characterization results, not a claim of clean static analysis. +Their `.test.wfl.log` files retain every diagnostic. + +| Probe | Static diagnostic | Interpretation | +|---|---|---| +| `list-parameter` | `List` differs from `List of Any`; a list argument also differs from `List`. | Inconsistent parser representation of the same collection annotation. | +| `container-parameter` | Cannot access `label` on non-container type `Item`. | Named container parameter is represented as a custom type, but the property resolver does not recognize that registered container. | +| `include-definition` | Included container type/metadata not found. | Static include traversal is absent; the definition exists when execution reaches it. | +| `parent-definition` | Container metadata not found in an included consumer. | Parent type symbols alone do not populate the analyzer's container registry. | +| `parent-after-call` | Container metadata not found after a parent action call. | Same metadata limitation; no runtime failure. | +| `parent-after-method` | Expected `Container` but found `Any`. | Conservative call effects widen a parent container binding seeded as mutable, even though this particular method leaves it intact. | +| `untyped-factory` | Typed `Text` property initialized from `Unknown`. | Intentional persistent-property safety diagnostic: the untyped function signature does not establish its input contract. | +| `typed-factory` | None. | Adding the existing `as text` parameter annotation establishes the contract and preserves the same successful call. | + +The final two files differ only by `with parameters label_value` versus +`with parameters label_value as text`. This distinguishes an authored static +contract gap from a runtime false positive. Successful tests alone cannot prove +that every untyped call will always satisfy a typed property. + +## Concrete causes and source anchors + +Source references below are in `WebFirstLanguage/wfl` at the revision above. + +1. `src/parser/stmt/containers.rs:64` parses a bare property `List` as + `Type::List(Type::Any)`. In contrast, method parameter parsing at + `src/parser/stmt/actions.rs:541` uses `colon_type_from_token` at line 45, + whose primitive cases omit `List`, leaving `Type::Custom("List")`. + Scriptorium uses the documented colon-style method form, not an invented + annotation syntax. A minimal reproduction is: + + ```wfl + create container Basket: + property entries: List defaults [] + action replace_entries needs replacement: List: + change entries to replacement + end + end + ``` + +2. `src/typechecker/mod.rs:10693` resolves dot-property access for + `ContainerInstance` and other concrete runtime types. A method parameter + annotated `incoming: Item` instead carries `Custom("Item")`. That custom + name is valid for nominal assignment compatibility + (`src/typechecker/mod.rs:11107`), but does not reach the registered + container's property lookup. The minimal failing diagnostic is produced by + `change saved_label to incoming.label` in a method with `incoming: Item`, + even when `Item` and its `Text` property are in the same file. + +3. `src/typechecker/mod.rs:7631` explicitly documents that the checker does + not parse includes. The runtime checks each included file before executing + its nested includes (`src/interpreter/mod.rs:9190` and `:9219`). + `snapshot_parent_scope` (`src/interpreter/mod.rs:5292`) transports variable + types and action signatures, but not the container metadata registry. + Consequently a later file can see `Container("Item")` without its field and + inheritance metadata. `escape_all_visible_mutable_state` + (`src/typechecker/mod.rs:1786`) further widens mutable imported bindings to + `Any` across an opaque method/constructor call; the local-container + declaration is registered immutable while the runtime snapshot uses the + runtime binding's mutability. The `parent-after-method` probe reproduces + exactly the application's `not a container type ... found Any` category. + +4. `are_declared_property_types_compatible` + (`src/typechecker/mod.rs:10975`) deliberately rejects `Any`/`Unknown` + sources for a concrete persistent property. Ordinary action compatibility + is more gradual; persistent properties retain their declared type on later + reads, so accepting an unchecked replacement would make those reads + unsound. Existing Rust regression coverage at + `tests/typechecker_container_contract_test.rs:177` specifically protects + this rule. Do not loosen it to silence Scriptorium factories. Where a + factory's parameter contract is known, adding the existing specific type + annotation is the appropriate application improvement. Dynamic list + construction and included action return inference may still need a separate + evidence-based checker improvement; assigning `Any` everywhere is not one. + +The container guide demonstrates colon-style properties, `needs` parameters, +and named container parameters (`docs/04-advanced-features/containers-oop.md`, +including its Task/TaskList example). The action guide documents the additive +`as text` form (`docs/03-language-basics/actions-functions.md:249`). +The CLI deliberately emits type diagnostics as nonfatal warnings before normal +execution (`src/main.rs:965`); the included-file path does the same. The +`error[ERROR]` presentation does not mean these tests were skipped or that the +runner discarded a nonzero result. It also does not make the diagnostic +limitations disappear. + +## Bounded follow-up plan + +The first two issues are suitable for a focused upstream change, separately +reviewed and tested before use: + +- Normalize the already documented bare `List` method-parameter annotation to + the same list type as a property. Reuse the property type grammar if extending + parameter annotations to nested lists, rather than maintaining divergent + parsers. Preserve existing case and custom-type rules; explicitly examine + legacy containers named `List` before changing their interpretation. +- Resolve `Custom(name)` through the registered container in value-member + contexts. Reuse inheritance/property lookup, preserve built-in handle and + temporal type rules, and continue rejecting unknown names and missing fields. + Check method calls through the same nominal value annotations as well. +- Use a WFL process driver to run minimal WFL fixtures and inspect captured + diagnostics. Positive fixtures must still execute and lose only the false + diagnostic; negative fixtures must retain mismatched element/property types, + missing fields, unrelated container arguments, and unsafe `Any`/`Unknown` + persistent-property assignments. Cover inherited named parameters and + unchanged built-in/custom name behavior. Register these WFL tests in the + existing gated runner; no new Rust/Python test scenario or driver is needed. + +Include-aware analysis is a larger change: preserving complete parent container +contracts is only part of it, because a main file is checked before a nested +include executes. A sound static dependency pass must respect include scope, +shadowing, inheritance, cycles, source budgets, and runtime binding effects +without executing application code. Flattening the application's include tree, +reordering definitions to fool the checker, marking unknown definitions as +containers, or broadly exempting `Any` are not sound small patches. + +These findings are not new functional prerequisites for the passing CMS flows. +They are real diagnostic limitations and authored contract gaps to track +explicitly; the application must not claim a clean type-analysis run. No new +upstream PR was started as part of this investigation. + +## No-Unlearning assessment + +The foundation's overarching invariant (`docs/wfl-foundation.md:101`) requires +beginner and production forms to be the same or joined by an additive path. +Making `List` and named-container annotations mean the same thing on properties +and parameters repairs that path. Adding a precise parameter annotation to a +known factory contract also follows it: the existing call form and property +model stay intact. Removing annotations, teaching different spellings merely +to evade diagnostics, flattening reusable modules, or asking authors to ignore +all type warnings would create habits that production users must unlearn. +Keeping the current limitations visible is an honest interim statement, not a +foundation exception or acceptance of those workarounds. diff --git a/docs/wfl-test-inventory.md b/docs/wfl-test-inventory.md new file mode 100644 index 0000000..b175dfa --- /dev/null +++ b/docs/wfl-test-inventory.md @@ -0,0 +1,240 @@ +# WFL test conversion inventory and runtime gates + +Status: conversion inventory and executable evidence, 2026-09-20. The WFL runner +and both tooling replacements pass against the reviewed source-built process +runtime; published-nightly/final-revision acceptance is tracked separately. +The original Python runner/tooling files were removed after all mapped WFL +cases passed and independent source review accepted the mapping. The user explicitly authorized their conversion; +this document does not request a separate layout migration. + +The acceptance basis is `testing.md`, the binding root policies, the attached +task, and the complete local `G:/repos/wfl/Docs/wfl-foundation.md` Version 2. +The task's quoted No-Unlearning Invariant matches that reference. No separate +foundation attachment was supplied to this task beyond the task text's local +reference. The replacement must keep one readable WFL testing path from a +single suite to the complete run, with real assertions, actionable diagnostics, +bounded child ownership, and no Python implementation of test behavior. + +## Existing executable coverage + +| Existing entry point | Scenarios and limits | Conversion obligation | +|---|---|---| +| `TestPrograms/util.test.wfl` | 25 tests: slug normalization; number/default and field helpers; form and cookie parsing; truncation; extension/stem parsing; config parsing; valid, missing, empty, malformed, fractional and out-of-range ports; installation input validation. | Preserve and discover it. | +| `TestPrograms/db.test.wfl` | 16 tests: schema repeat initialization, users/settings/posts/pages/sessions/media/login-attempt helpers, expiry, legacy CSRF-column upgrade, installation state/application. Uses memory SQLite. | Preserve while adding file-backed ORM/migration/recovery coverage. | +| `TestPrograms/auth.test.wfl` | 3 tests: own CSRF token, missing token, session token binding. | Preserve; does not replace real login/authorization tests. | +| `TestPrograms/scribe.test.wfl` | 10 tests: Markdown blockquotes/code fences, safe-marker filters, escaping, suffix escaping, nested blockquotes. | Preserve and keep upstream ownership. | +| `TestPrograms/render.test.wfl` | 6 tests: theme default, configured and external roots, traversal rejection. | Preserve; does not prove HTTP rendering journeys. | +| `lib/scribe/tests/scribe.test.wfl` | 83 pinned upstream tests; writes `build/` fixtures. | Copy the pinned source without `.git`, existing `build`, or caches; run from the copy with a fresh `build/`; remove the copy. Never patch the submodule in place. | +| `tests/tooling/test_run_tests.py` | 8 regression methods, mapped below. | All fixtures, assertions, and driver logic must become WFL. | +| `tests/tooling/test_repo_hygiene.py` | 28 regression methods, mapped below. | WFL may invoke the unchanged Python hygiene checker as its implementation subject; Python must not own test behavior. | +| `tests/integration/test_server_port.py` | 3 real HTTP startup tests, mapped below. | Preserve all existing cases and cleanup, then extend actual CMS journeys. | + +The original `scripts/run_tests.py` recursively discovers and lexically sorts regular +`TestPrograms/**/*.test.wfl` files, resolves its own repository root independent +of the caller's directory, resolves the selected interpreter, and runs suites +sequentially with repository cwd. Default suite timeout is 120 seconds. It +continues after failures and timeouts, preserves interpreter stdout/stderr, +reports every result and the total, and exits 1 on suite failure. Empty +discovery, bad interpreter, bad timeout, and missing requested Scribe sources +fail before running tests. Setup/cleanup failures exit 2; interruption exits +130. There are no retries. This is the historical contract inventory. + +`scripts/run_tests.wfl` now discovers every maintained group by default, +including pinned Scribe. Focused commands use `--group application`, `tooling`, +`integration`, `runtime`, `examples`, or `scribe`; `--include-scribe` remains compatible +when adding Scribe to a focused run. The same-runtime default uses +`current_executable`; native launch resolves a bare `--wfl` name, then the runner +passes that exact resolved runtime to every suite as `args[0]`. Suite failures +and timeouts exit 1; setup/cleanup failures exit 2. Captured stdout and stderr +contents are preserved in the runner's output, with stderr explicitly labeled. +WFL owns interruption handling; exact Python KeyboardInterrupt formatting and +exit 130 are not a promised cross-runtime CLI contract. Child-tree ownership +and bounded cleanup remain required. Timeout arguments are finite positive JSON +numbers up to one year; spell `.5` and `+1` as `0.5` and `1`. + +## Runner regression mapping + +The first eight tests in `tests/tooling/runner.test.wfl` implement the eight +requirements below in the same order. The ninth verifies the complete default +discovers all maintained groups plus Scribe and excludes helper files. +They passed 9/9 on Windows with process runtime commit `a32c74f1`. +Direct argument lists avoid shell quoting; no fake Python interpreter remains +in the replacement fixtures. + +| Python test method | Equivalent WFL test and required observation | +|---|---| +| `test_discovers_nested_suites_in_order_and_uses_repository_cwd` | WFL creates nested `a.test.wfl`, `z.test.wfl`, and ignored `example.wfl`; launches the copied WFL runner from outside its fixture root; asserts exactly `a`, then `z`, and the fixture root as each suite's cwd. Requires child cwd support. | +| `test_failing_suite_preserves_diagnostics_and_later_suite_runs` | A real WFL child fails intentionally with recognizable diagnostics; a second suite succeeds. Assert runner exit 1, both executions, failed suite name, and preserved stderr as well as stdout. | +| `test_timeout_fails_and_later_suite_still_runs` | WFL child waits longer than `--timeout 1`; next child writes a completion marker. Assert timeout diagnostic, both starts, second completion, runner failure, and absence of owned child processes afterward. | +| `test_empty_suite_set_is_an_error` | Empty fixture `TestPrograms/`; assert nonzero exit and empty-suite diagnostic. | +| `test_missing_interpreter_is_an_error` | Supply a nonexistent executable; assert nonzero exit and no fixture starts. | +| `test_nonpositive_and_nonfinite_timeout_are_rejected` | Parameterize `0`, `-1`, `nan`, and `inf`; each fails before any child. Retain every case. | +| `test_missing_scribe_source_is_an_error` | Request Scribe with missing source/test files; assert nonzero exit and Scribe diagnostic before running suites. | +| `test_upstream_suite_runs_in_disposable_copy_with_build_directory` | WFL creates minimal Scribe source/test and an original `build/existing.txt`. Assert the upstream fixture runs after local suites, has cwd distinct from original Scribe, sees fresh `build/`, its copy is removed, and original file bytes remain unchanged. | + +The old fake interpreter is Python and uses an OS-specific shell launcher. +Neither fixture is eligible to survive this conversion. Real WFL fixture +programs can supply success, failure, delay, cwd, and marker behavior; a narrow +WFL fixture protocol can provide controlled runner observations without an +executable shell test implementation. + +## Hygiene regression mapping + +`tests/tooling/hygiene.test.wfl` implements the 28 rows below, in the same order, +and passed 28/28 on Windows. Common setup is WFL: make a unique disposable directory, run `git init`, +write every profile-required file and synthetic sources, stage them, and add +the approved Scribe gitlink with `git update-index --cacheinfo`. The checker +receives `--root` explicitly, so these tests do not require changing the parent +process directory. Capture its exit code and both output streams. WFL must +generate binary fixtures, edit profiles, inspect results, and clean up even +when an assertion fails. No real repository index is a fixture. + +| Python test method | Equivalent WFL fixture and assertion | +|---|---| +| `test_clean_checkout_and_uninitialized_submodule_pass` | Baseline synthetic index with opaque Scribe gitlink; exit 0 and success text. | +| `test_new_untracked_root_file_is_checked_before_staging` | Add unstaged `scratch.md`; exit 1 and exact root-path diagnostic. | +| `test_new_untracked_root_directory_is_rejected` | Add `notes/design.md`; exit 1 and root diagnostic. | +| `test_legitimate_new_docs_and_binary_source_assets_pass` | Add maintained docs, synthetic WOFF2 bytes, PNG bytes under docs; exit 0. | +| `test_ignored_untracked_runtime_data_is_left_alone` | Write ignored `.db` and `.env`; exit 0 and unchanged database bytes. | +| `test_force_tracked_ignored_database_is_rejected` | Force-stage `app/fixture.db`; exit 1 and artifact diagnostic. | +| `test_repository_ignores_sqlite_databases_and_sidecars` | Copy actual `.gitignore`; create all 12 `.db`/`.sqlite`/`.sqlite3` combinations with empty, `-journal`, `-wal`, `-shm` suffixes; exit 0. | +| `test_case_variants_and_nested_cache_are_rejected` | Disable fixture ignores; separately test `app/site.SQLITE3-WAL`, `docs/server.LOG`, and `scripts/__PyCache__/bad.txt`; exact artifact diagnostic for each. | +| `test_secret_names_are_rejected_at_any_depth` | Separately test `.env.production`, `id_rsa`, and uppercase `.KEY` at the original depths; exit 1 for each. | +| `test_upload_image_cannot_use_source_asset_exception` | PNG under `static/uploads/`; exit 1 and runtime-state diagnostic. | +| `test_only_empty_declared_upload_placeholder_is_allowed` | Nonempty `static/uploads/.gitkeep`; exit 1. | +| `test_removed_and_empty_required_policy_fail` | Delete then recreate empty `GOVERNANCE.md`; both fail with their specific required/empty diagnostic. | +| `test_ignored_required_policy_cannot_pass_by_existing_locally` | Remove the policy from fixture index and ignore it; local existence still fails required-file check. | +| `test_worktree_link_changes_are_checked_without_staging` | Unstaged README link to nonexistent doc; exit 1 for candidate-tree link. | +| `test_link_to_ignored_local_file_cannot_hide_broken_clone` | Link to ignored local `.private` file; exit 1 even though it exists locally. | +| `test_links_to_new_candidates_directories_and_encoded_paths_pass` | New spaced-name doc; percent-encoded/fragment, directory, angle-bracket, and reference links all pass. | +| `test_reference_and_image_link_destinations_are_checked` | Missing image and reference-link definitions; assert both diagnostics in one failing result. | +| `test_anchors_urls_fences_and_submodule_links_are_skipped` | Anchor, HTTPS, mailto, uninitialized submodule destination, backtick fence and tilde fence examples all pass. | +| `test_links_cannot_escape_checkout` | README `../outside.md`; exit 1 for checkout escape. | +| `test_submodule_contents_are_not_recursively_inspected` | Synthetic `.db` and `.env` within Scribe gitlink directory; exit 0. | +| `test_scribe_cannot_be_replaced_with_copied_files` | Remove gitlink, create copied source; exit 1 for missing required gitlink. | +| `test_another_gitlink_cannot_bypass_artifact_inspection` | Add opaque `lib/other` gitlink; exit 1 for unapproved gitlink. | +| `test_indexed_symlink_is_rejected_even_when_checked_out_as_text` | `git hash-object -w` fixture content; index mode `120000` via cacheinfo while physical file is text; exit 1 for file type. Works without Windows symlink privileges. | +| `test_malformed_profile_fails_closed` | Invalid TOML `schema = [`; exit 2 and `HYGIENE-ERROR`. | +| `test_missing_profile_fails_closed` | Remove profile; exit 2 and profile-read error. | +| `test_unknown_profile_key_fails_closed` | Replace `allowed-dirs` with `allowed-directories`; exit 2 for unknown/missing keys. | +| `test_profile_cannot_use_escape_paths` | Replace a required architecture path with `../ARCHITECTURE.md`; exit 2 for nonrelative path. | +| `test_git_failure_fails_closed` | Move only fixture `.git` away; exit 2 and Git diagnostic. | + +## HTTP port regression mapping + +WFL setup must copy only application code, pinned Scribe source and installer +template into a disposable site; never copy database, uploads, config or Git +metadata. It creates a parent `.wflcfg` restricting the runtime to loopback, +including for the absent site-config case. A site config adds a disposable +`data_dir`. It owns the direct server child and its diagnostics, checks early +exit, polls `/install` for at most 20 seconds, uses bounded individual HTTP +requests, and cleans up on success, assertions, errors and timeouts. It must +refuse occupied ports without stopping an existing service. The original +driver requests an OS-assigned port for the configured case and checks that +8080 is also free, so a hardcoded-port regression produces useful output. + +| Python test method | Equivalent WFL case | +|---|---| +| `test_configured_port_serves_installer_and_updates_startup_urls` | Explicit non-8080 available port; GET installer on it and assert both startup URL messages use it. | +| `test_missing_port_setting_defaults_to_8080` | Site config exists but port setting omitted; installer and both startup URLs use 8080. | +| `test_missing_config_file_defaults_to_8080` | Site `.wflcfg` absent; inherited runtime loopback config still protects fixture; app falls back to 8080. | + +Every case asserts HTTP 200, `Content-Type` containing `text/html`, body text +`Set up your site` and `Scriptorium`, form action `/install`, and a named +`csrf_token` input. It asserts the exact configured `Scriptorium is running` +and `First run` URLs in the actual child log. Unit tests already cover invalid +port values; new HTTP cases should extend this coverage, not reduce it. + +## Runtime evidence and upstream remedies + +The following table preserves the initial Red findings, before the upstream +remedies; it is not the current source-built candidate's capability status. +Source inspected: WFL revision `cb1dadaad96939a4450a6eb2b3a6a51678035b7f`. +Executable probes were run on Windows with official nightly WFL `26.9.12` +from the locally extracted release, with that binary's directory first on +`PATH` so child runtimes match. These are native Windows results; the +Blacksmith Linux nightly results and resolved Docker digest must be recorded +separately. All fixtures contain synthetic data only. + +| Capability | Verified result and effect | +|---|---| +| Filesystem | WFL creates parent directories, writes/reads/copies files, recursively lists paths, and deletes its own tree. `call remove_dir with path and yes` works. `list files recursively` is not sorted; the runner must sort explicitly. | +| Arguments and locations | `args`, `script_path`, `script_directory`, `current_directory` work. Runtime config is located from source directory; process cwd stays inherited. | +| Foreground process | `execute command` with argv returns `output`, `error`, `exit_code`, `success`. It preserves controlled child stdout/stderr and failures. Current source applies the overall runtime deadline, not an independent configurable timeout for each child. | +| Background lifecycle | Direct `spawn command`, `process ... is running`, `kill process`, and exit retrieval work. `finally` kills the owned child after both a runtime error and a failed assertion. | +| Async diagnostics | Confirmed blocker: `read output from process` exposes only stdout, although stderr is buffered internally. Waiting removes the handle and makes final output inaccessible. Capture tasks are not joined by that wait. A runner needs a complete outcome with both streams after reaping, without races or lost errors. | +| Child cwd | Confirmed missing primitive: neither spawn/execute grammar nor runtime implementation accepts a working directory, and there is no process chdir API. Launching an absolute WFL fixture path still uses the parent cwd. This prevents the existing disposable Scribe and real application fixtures from running unchanged with their required relative paths. Add a native per-launch cwd option; retain direct ownership of the actual child. | +| HTTP requests | Native POST with request headers/body returns status, body, headers, `ok`; ordinary response cookies are observable. | +| Redirect handling | Confirmed blocker: client follows redirects automatically and returns only the final response. A synthetic login POST giving 302 plus a session cookie becomes 200 with neither intermediate status nor cookie. Add explicit no-follow behavior or an equally complete redirect-chain result; preserve multiple header values. | +| Assertions/failure exit | `wfl --test` returns 1 for a genuine failed expectation. `expect` outside test mode errors instead of asserting. Documentation's `exit with code 1` does not parse as a supported exit-code construct; only successful `exit program` is implemented. A test-mode runner is possible, but exact ordinary-script exit-code compatibility needs a runtime remedy. This syntax mismatch is not behavioral Red evidence. | +| Error handling | `try`/`when error` expose `error_message`; `finally` runs cleanup and the original failure still propagates. No intentional failure is swallowed into a pass. | + +The accepted foundation path cannot require people to learn shell quoting, +synthetic division-by-zero as a normal error API, or modified copies of Scribe +and the application just to run their tests. Absolute include paths do not +repair Scribe's relative `build/` or the app's cwd-based configuration/data +resolution. Shell wrappers would also change which process WFL owns and kills +on Windows. Neither is an accepted replacement for the missing primitive. + +The original `a81ecf9` capability gates under `tests/runtime/` deliberately +exposed missing runtime behavior: + +```sh +wfl --test tests/runtime/process-capabilities.test.wfl +wfl --test tests/runtime/http-redirect-capability.test.wfl +``` + +The process gate has four valid scenarios: cwd isolation, output after reaping, +stderr preservation, and error-path cleanup. On Windows 26.9.12 it reports +**1 passed / 3 failed, exit 1**; error-path cleanup passes. The HTTP gate reports +**0 passed / 1 failed, exit 1**, specifically `Expected 200 to equal 302`. +The HTTP probe requires free loopback port 41868 and checks occupancy before +starting its own server. These are historical upstream capability failures, +not application defects or replacements for the existing suites. The reviewed +upstream remedies make the current capability suites pass on the combined +candidate, and the complete runner now includes them by default. See +[the verification record](orm-verification.md) for published-runtime results. + +Relevant runtime source locations: `src/parser/stmt/processes.rs` (launch +grammar), `src/interpreter/mod.rs` (`spawn_process`, `read_process_output`, +`wait_for_process`, `kill_process`, `http_client`), and +`src/parser/stmt/actions.rs` (`parse_exit_statement`). Remedies need upstream +regression tests, a published runtime, and reruns against an immutable nightly +image before the complete WFL-only runner can be claimed verified. + +## CI and documentation conversion checklist + +The converted Governance workflow provisions WFL on Blacksmith Ubuntu and GitHub +Windows, records the published release URL/asset/SHA256/version, runs +`wfl scripts/run_tests.wfl --group tooling`, and retains the hygiene checker. +Python remains only for that non-test checker implementation. + +The WFL tests workflow pulls `bsbyrdwfl/wfl:nightly`, resolves its digest, +records runtime/Scriptorium/Scribe versions, and runs +`wfl --execution-timeout 1200 scripts/run_tests.wfl` +in that resolved container. A read-only source mount is copied into a disposable +writable checkout. Suite, Scribe-copy and HTTP orchestration are WFL. The Update +Scribe generated PR checklist uses these same commands. Official nightly +26.9.16 contains the reviewed upstream remedies, including the explicit finite +invocation budget; final remote acceptance is tracked in `orm-verification.md`. + +The hygiene profile now requires the WFL runner/helpers/config and both tooling +suites. Guidance uses the same complete/focused commands. The examples directory +houses the explicitly requested executable ORM/migration progression; the +application and test include layouts are retained. + +The unchanged original Python tooling tests passed **36/36** before removal; +the replacement focused command passed **2 suites, 37/37 tests**. Its intentional +WFL assertion fixture returns runner exit 1, preserves diagnostics, and allows +later suites to complete. Its owned child/grandchild timeout fixture proves no +late marker survives cleanup. Fixture parser mistakes encountered during +development are not counted as behavioral Red evidence. Full candidate and +remote deliberate-failure evidence remain separate final acceptance steps. + +Independent review checked every row above against executable WFL scenarios +before removing the Python versions. The complete official-image run at +`58362064` included pinned Scribe, file-backed ORM/migrations/recovery and HTTP +workflows: 41 functional suites passed and only the deliberate real WFL assertion +failed, propagating exit 1 to GitHub Actions. The temporary suite is now removed. +Repository hygiene and both Governance platforms passed. See +`orm-verification.md` and PR #16 for the isolated proof and final clean-head CI. diff --git a/docs/wfl-tooling-validation.md b/docs/wfl-tooling-validation.md new file mode 100644 index 0000000..164aae8 --- /dev/null +++ b/docs/wfl-tooling-validation.md @@ -0,0 +1,111 @@ +# WFL tooling conversion evidence + +The mapping in [wfl-test-inventory.md](wfl-test-inventory.md) preserves all eight +original runner requirements and all 28 hygiene requirements. The resulting +suite, its fixtures and helpers, and the complete runner are WFL. The unchanged +Python hygiene utility is only an implementation subject invoked by WFL. + +## Local results, 2026-09-20 + +`wfl scripts/run_tests.wfl --group tooling` passed both suites: + +| Suite | Result | Meaning | +|---|---|---| +| `tests/tooling/runner.test.wfl` | 9/9 | Eight original requirements plus complete default discovery of maintained suite groups and Scribe | +| `tests/tooling/hygiene.test.wfl` | 28/28 | Every inventory row, including binary source assets, all SQLite sidecars, synthetic Git indexes/gitlinks, ignored files, links and fail-closed setup errors | + +The process-only source-built runtime at upstream `a32c74f1` passed first; the +combined runtime `ae5395d9bd215d0d9fa1c039e666f03c8897cf88` passed after the hygiene +profile changed to require WFL files. Both report WFL `26.9.12`; the published +26.9.12 release does not contain these changes. Combined executable SHA256: +`76F612CA1BBAF4484B2CC2BDA86D876588BC765DE9C38B7DAA8951BAD143B911`. +Local detailed output is ignored at `target/tooling-combined-results.txt`. + +The runner suite executes a deliberately failing WFL expectation and a distinct +runtime-error fixture. It requires runner exit 1, both stdout and stderr +diagnostics, and successful execution of the later suite. The timeout case owns +a WFL child/grandchild, verifies the descendant started, and asserts its delayed +write never occurs after the hard timeout. Scribe tests verify a separate cwd, +fresh build directory, exclusion of Git/cache metadata, removal of the temporary +copy, and unchanged original bytes. No retry or expected-failure wrapper hides +these outcomes. Fixture syntax errors during development are not Red evidence. + +Independent root-agent source review accepted the runner/helper and the +37-case behavior mapping before removal of the Python runner and tooling +suites. Review is technical; Maintainer acceptance remains separate. + +## Changes to the command contract + +A second independent review found one missing edge: output printed before a +timeout was discarded by native process completion. Strengthening the WFL +timeout case produced **8/9, exit 1**, specifically because the recognizable +pre-timeout diagnostic was absent; later suites still ran. The corresponding +upstream WFL test-first commit `012e7c89` also failed its intended assertion. +The runtime remedy drains both bounded streams after termination and includes +their contents in the same typed timeout error, with a separate one-second +drain limit. The strengthened Scriptorium runner suite then returned **9/9**. +Independent source review accepted that remedy. The final combined-runtime +result is recorded below; published-runtime acceptance is tracked separately. + +The complete command includes all groups and pinned Scribe by default. Use +`--group` for an explicitly focused run. The default executable is the exact +runtime which launched the runner; `--wfl` still overrides it. Captured stderr +content is labeled in the runner's output, since WFL's display facility writes +stdout. The runtime handles user interruption and owns shutdown cleanup; +Python's specific KeyboardInterrupt message/status is not a portable promise. +Timeouts use finite positive JSON-number notation up to one year. + +The subprocess wait limit is separate from WFL's whole-invocation execution +budget. Official WFL 26.9.14 clamps `timeout_seconds` to at most 300 seconds; +zero becomes one second, and that version provides no CLI timeout override. +Accordingly `scripts/.wflcfg` states 300 instead of the ineffective 3600. +That correction changes no effective behavior. Official WFL 26.9.16 adds an +explicit finite invocation option, used by the complete command as +`wfl --execution-timeout 1200 scripts/run_tests.wfl`; its verified publication +and final remote acceptance are tracked in `orm-verification.md`. Each child +still has its own runtime deadline and bounded subprocess wait. The runner's +`--timeout` option does not extend either runtime deadline. + +The frozen complete suite at Scriptorium +`3bc7b4faaf9c000047d940f0e420067b777db5c9` passed **41/41 suites** on combined WFL +`df6ad9a2`. The new `target/full-candidate-final.log` file's creation and final +write timestamps span approximately **160.5 seconds**, below the effective +300-second bound. This is file-metadata timing, not a separate benchmark. +The portability review confirmed that published nightly asset patterns match +the extracted executable layouts, the Docker root override can write its +disposable `/work` copy, and native WFL failure statuses propagate through both +governance platforms and the complete-suite Docker command. Exact-revision +remote Green remains required; this source review does not claim it. + +## Final acceptance record + +The independent 2026-09-20 source audit rechecked all eight original runner, +28 hygiene and three HTTP-port requirements against the replacement WFL files. +It found no missing conversion case or non-WFL scenario, fixture, assertion or +driver. The five original WFL application suites are unchanged from baseline +`4ec5c88d9e4ae27041599ad8293a28130b56fbf2`; Scribe's gitlink and checkout remain +`93d62af5a6ed6c3ce257ef888107fc3ca1e2dc1d`. Final discovery is 23 application, +three tooling, eight integration, five runtime, one example and one Scribe suite: +41 total. The temporary deliberate CI suite raised this to 42 for the isolated +remote failure proof and has been removed. +The audit also reviewed owned-process cleanup, descendant timeout markers, +failure diagnostics and continuation, disposable Scribe copies, real HTTP and +backup/restore boundaries, migration concurrency/interruption, and the executable +API progression. Two stale command descriptions were corrected. This source +review does not replace the final execution and publication gates below. + +The full application/ORM/migration/recovery/HTTP/example suite and final-revision +remote jobs are recorded with the overall PR. Governance retains Blacksmith +Linux and GitHub-hosted Windows tooling coverage with explicit WFL provisioning. +The complete nightly job still pulls `bsbyrdwfl/wfl:nightly`, resolves its digest, +records runtime and source revisions, and invokes the WFL runner in a writable +disposable copy of a read-only checkout. Python executes only the hygiene +checker subject. A source-built local pass does not establish a published +nightly pass. Official-image run +[35513765769](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35513765769) +at `58362064` completed with **41 functional passes and only the deliberate +assertion failure**, exit 1. The exact marker, runtime provenance and cleanup +were independently verified before removing that file. Both Governance +platforms passed all 41 tooling assertions and the six focused gates passed. +`orm-verification.md` records this evidence; the final clean 41-suite revision +and its exact-head CI result are retained in PR #16's Validation section. diff --git a/examples/orm/.wflcfg b/examples/orm/.wflcfg new file mode 100644 index 0000000..31a3e21 --- /dev/null +++ b/examples/orm/.wflcfg @@ -0,0 +1,3 @@ +timeout_seconds = 180 +execution_logging = false +debug_report_enabled = false diff --git a/examples/orm/migrations/20260920000000_initial.wfl b/examples/orm/migrations/20260920000000_initial.wfl new file mode 100644 index 0000000..3871fc1 --- /dev/null +++ b/examples/orm/migrations/20260920000000_initial.wfl @@ -0,0 +1,62 @@ +// Immutable version-specific schema, independent of the current model module. +include from "../../../lib/orm/migrations.wfl" + +define action called example_v1_authors: + create new OrmField as author_key: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes + end + create new OrmField as name_field: + field_name is "name" + unique_value is yes + end + create new OrmModel as authors: + table_name is "example_authors" + fields is [author_key and name_field] + end + return authors +end action + +define action called example_v1_notes: + create new OrmField as note_key: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes + end + create new OrmField as title_field: + field_name is "title" + end + create new OrmField as owner_field: + field_name is "author_id" + value_type is "identity" + nullable is yes + end + create new OrmField as urgent_field: + field_name is "urgent" + value_type is "boolean" + has_default is yes + default_value is no + end + create new OrmModel as notes: + table_name is "example_notes" + fields is [note_key and title_field and owner_field and urgent_field] + end + return notes +end action + +define action called example_initial_migration with parameters source_text: + store authors_table as orm_schema_table of (example_v1_authors) + store notes_table as orm_schema_table of (example_v1_notes) + create new OrmMigration as initial: + migration_id is "20260920000000" + migration_name is "Create authors and notes" + up_steps is [(orm_migration_create_table of authors_table) and (orm_migration_create_table of notes_table)] + managed_after is [authors_table and notes_table] + irreversible_reason is "dropping the initial tables would destroy saved notes" + source_text is source_text + end + return initial +end action diff --git a/examples/orm/migrations/20260920000100_author_index.wfl b/examples/orm/migrations/20260920000100_author_index.wfl new file mode 100644 index 0000000..37c12ca --- /dev/null +++ b/examples/orm/migrations/20260920000100_author_index.wfl @@ -0,0 +1,36 @@ +include from "20260920000000_initial.wfl" + +define action called example_index_migration with parameters source_text: + store notes_before as example_v1_notes + create new OrmIndex as owner_index: + index_name is "example_notes_author" + field_names is ["author_id"] + end + create new OrmModel as notes_after: + table_name is "example_notes" + fields is notes_before.fields + indexes is [owner_index] + end + store authors_table as orm_schema_table of (example_v1_authors) + create new OrmMigration as indexed: + migration_id is "20260920000100" + migration_name is "Index note ownership" + up_steps is [(orm_migration_create_index of (orm_schema_table of notes_after) and owner_index)] + down_steps is [(orm_migration_drop_index of (orm_schema_table of notes_after) and owner_index)] + managed_before is [authors_table and (orm_schema_table of notes_before)] + managed_after is [authors_table and (orm_schema_table of notes_after)] + source_text is source_text + end + return indexed +end action + +define action called example_registry: + store migration_dir as path_join of current_directory and "examples" and "orm" and "migrations" + open file at (path_join of migration_dir and "20260920000000_initial.wfl") for reading as initial_file + wait for store initial_source as read content from initial_file + close file initial_file + open file at (path_join of migration_dir and "20260920000100_author_index.wfl") for reading as index_file + wait for store index_source as read content from index_file + close file index_file + return [(example_initial_migration of initial_source) and (example_index_migration of index_source)] +end action diff --git a/examples/orm/models.wfl b/examples/orm/models.wfl new file mode 100644 index 0000000..f54d011 --- /dev/null +++ b/examples/orm/models.wfl @@ -0,0 +1,60 @@ +include from "migrations/20260920000100_author_index.wfl" + +// Current models can evolve; historical migration modules never call them. +define action called example_authors: + create new OrmField as author_key: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes + end + create new OrmField as name_field: + field_name is "name" + unique_value is yes + end + create new OrmModel as authors: + table_name is "example_authors" + fields is [author_key and name_field] + end + return authors +end action + +define action called example_notes: + create new OrmField as note_key: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes + end + create new OrmField as title_field: + field_name is "title" + end + create new OrmField as owner_field: + field_name is "author_id" + value_type is "identity" + nullable is yes + end + create new OrmField as urgent_field: + field_name is "urgent" + value_type is "boolean" + has_default is yes + default_value is no + end + create new OrmIndex as owner_index: + index_name is "example_notes_author" + field_names is ["author_id"] + end + create new OrmModel as notes: + table_name is "example_notes" + fields is [note_key and title_field and owner_field and urgent_field] + indexes is [owner_index] + end + create new OrmRelationship as author_relation: + relation_name is "author" + local_field is "author_id" + related_model is example_authors + related_field is "id" + end + notes.relate(author_relation) + return notes +end action diff --git a/examples/orm/progression.test.wfl b/examples/orm/progression.test.wfl new file mode 100644 index 0000000..414efa1 --- /dev/null +++ b/examples/orm/progression.test.wfl @@ -0,0 +1,68 @@ +// Run from the repository root: wfl --test examples/orm/progression.test.wfl +include from "models.wfl" + +store artifact_dir as path_join of current_directory and "target" and "test-artifacts" and "orm-example" +create directory at artifact_dir +store database_path as path_join of artifact_dir and "example.db" +store fixture_session as orm_open of database_path +store conn as fixture_session.connection +store migrations as example_registry +try: + // Production and first examples both use explicit versioned migrations. + call orm_migrate with fixture_session and migrations and "latest" + describe "One API from the first record to schema evolution": + test "begin with a model, an explicit migration and a saved record": + store session as orm_borrow of conn + store notes as example_notes + store draft as orm_record of notes + call orm_set with draft and "title" and "My first note" + store saved as orm_save of session and draft + store found_record as orm_find_key of session and notes and (orm_get of saved and "id") + expect orm_get of found_record and "title" to equal "My first note" + expect orm_get of found_record and "urgent" to equal no + expect orm_get of found_record and "author_id" to equal nothing + expect orm_find_key of session and notes and "9223372036854775807" to equal nothing + end test + + test "compose the same records and queries inside a transaction": + store session as orm_borrow of conn + store notes as example_notes + in transaction on conn: + store author as orm_record of (example_authors) + call orm_set with author and "name" and "Ada" + store saved_author as orm_save of session and author + store note as orm_record of notes + call orm_assign with note and [(orm_value of "title" and "Prepare release") and (orm_value of "urgent" and yes) and (orm_value of "author_id" and (orm_get of saved_author and "id"))] + store saved_note as orm_save of session and note + end transaction + store requested as orm_query of notes + call orm_where with requested and (orm_all of [(orm_compare of "urgent" and "equal" and yes) and (orm_compare of "title" and "contains" and "release")]) + call orm_order_by with requested and "title" and "ascending" + call orm_page with requested and 20 and 0 + store found_notes as orm_find of session and requested + expect length of found_notes to equal 1 + store before_queries as session.query_count + call orm_load with session and found_notes and "author" + expect session.query_count minus before_queries to equal 1 + store loaded_author as orm_related of found_notes[0] and "author" + expect orm_get of loaded_author and "name" to equal "Ada" + expect session.query_count minus before_queries to equal 1 + end test + + test "plan and reverse an index migration while preserving records": + store session as orm_borrow of conn + store before_count as orm_count of session and (orm_query of (example_notes)) + call orm_rollback_count with session and migrations and 1 + store planned as orm_migration_plan of session and migrations and "latest" + expect length of planned.pending to equal 1 + expect orm_count of session and (orm_query of (example_notes)) to equal before_count + call orm_migrate with session and migrations and "latest" + store finished_plan as orm_migration_plan of session and migrations and "latest" + expect length of finished_plan.pending to equal 0 + expect orm_count of session and (orm_query of (example_notes)) to equal before_count + end test + end describe +finally: + call orm_close with fixture_session + delete file at database_path +end try diff --git a/lib/orm/connections.wfl b/lib/orm/connections.wfl new file mode 100644 index 0000000..3ab9a0c --- /dev/null +++ b/lib/orm/connections.wfl @@ -0,0 +1,146 @@ +include from "queries.wfl" + +create container OrmSession: + property connection + property owns_connection: Boolean defaults no + property closed: Boolean defaults no + property query_count: Number defaults 0 + property last_sql: Text defaults "" + property last_parameter_count: Number defaults 0 + + action record_query needs sql_text: Text, parameter_total: Number: + change query_count to query_count plus 1 + change last_sql to sql_text + change last_parameter_count to parameter_total + end + action mark_closed: + change closed to yes + end +end + +define action called orm_open with parameters database_path: + open database at ("sqlite://" with database_path) as conn + create new OrmSession as session: + connection is conn + owns_connection is yes + end + return session +end action + +define action called orm_borrow with parameters conn: + create new OrmSession as session: + connection is conn + end + return session +end action + +define action called orm_close with parameters session: + check if session.closed is equal to no: + check if session.owns_connection: + store conn as session.connection + close database conn + end check + session.mark_closed() + end check + return yes +end action + +define action called orm_require_open with parameters session: + check if session.closed: + call orm_fail with "connection" and "this ORM session is closed; open or borrow a live connection" + end check + return yes +end action + +define action called orm_safe_diagnostic with parameters diagnostic and bound_values: + store safe_text as diagnostic + for each bound_value in bound_values: + check if (typeof of bound_value) is equal to "Text": + check if (length of bound_value) is greater than 0: + change safe_text to replace bound_value with "[redacted]" in safe_text + end check + end check + end for + return safe_text +end action + +define action called orm_read_sql with parameters session and compiled: + store valid as orm_require_open of session + store conn as session.connection + session.record_query(compiled.sql_text, (length of compiled.bound_values)) + try: + return query conn with compiled.sql_text and parameters compiled.bound_values + when error: + store diagnostic as orm_safe_diagnostic of error_message and compiled.bound_values + call orm_fail with "query" and (diagnostic with "; verify the model, migration status, and database access; assigned values are omitted") + end try +end action + +define action called orm_write_sql with parameters session and compiled: + store valid as orm_require_open of session + store conn as session.connection + session.record_query(compiled.sql_text, (length of compiled.bound_values)) + try: + return execute conn with compiled.sql_text and parameters compiled.bound_values + when error: + store diagnostic as orm_safe_diagnostic of error_message and compiled.bound_values + call orm_fail with "write" and (diagnostic with "; verify constraints, migration status, and database access; assigned values are omitted") + end try +end action + +// Explicit SQL escape hatch. Identifiers/expressions are authored SQL; all +// runtime values belong in bound_values. Native WFL rejects transaction-control +// statements; use the same connection in a native in-transaction block instead. +define action called orm_sql_query with parameters session and sql_text and bound_values: + return orm_read_sql of session and (orm_sql of sql_text and bound_values and "unknown") +end action + +define action called orm_sql_execute with parameters session and sql_text and bound_values: + return orm_write_sql of session and (orm_sql of sql_text and bound_values and "unknown") +end action + +define action called orm_find with parameters session and requested: + store compiled as orm_compile_select of requested and "records" + store native_rows as orm_read_sql of session and compiled + store mapped_rows as [] + for each native_row in native_rows: + push with mapped_rows and (orm_map_row of requested.model_spec and native_row) + end for + return mapped_rows +end action + +define action called orm_first with parameters session and requested: + create new OrmQuery as first_query: + model_spec is requested.model_spec + predicate is requested.predicate + selected_fields is requested.selected_fields + ordering is requested.ordering + page_limit is 1 + page_offset is requested.page_offset + end + store mapped_rows as orm_find of session and first_query + check if (length of mapped_rows) is greater than 0: + return mapped_rows[0] + end check + return nothing +end action + +define action called orm_find_key with parameters session and model_spec and identity_value: + store key_spec as orm_primary_field of model_spec + store requested as orm_query of model_spec + call orm_where with requested and (orm_compare of key_spec.field_name and "equal" and identity_value) + return orm_first of session and requested +end action + +define action called orm_count with parameters session and requested: + store compiled as orm_compile_select of requested and "count" + store native_rows as orm_read_sql of session and compiled + store first_row as native_rows[0] + return first_row["orm_total"] +end action + +define action called orm_exists with parameters session and requested: + store compiled as orm_compile_select of requested and "exists" + store native_rows as orm_read_sql of session and compiled + return (length of native_rows) is greater than 0 +end action diff --git a/lib/orm/migrations.wfl b/lib/orm/migrations.wfl new file mode 100644 index 0000000..20e6799 --- /dev/null +++ b/lib/orm/migrations.wfl @@ -0,0 +1,531 @@ +include from "schema.wfl" + +create container OrmMigrationStep: + property operation: Text defaults "sql" + property before_table defaults nothing + property after_table defaults nothing + property source_fields: List defaults [] + property target_fields: List defaults [] + property column_spec defaults nothing + property index_spec defaults nothing + property sql_text: Text defaults "" + property bound_values: List defaults [] +end + +create container OrmMigration: + property migration_id: Text defaults "" + property migration_name: Text defaults "" + property up_steps: List defaults [] + property down_steps: List defaults [] + property irreversible_reason: Text defaults "" + property managed_before: List defaults [] + property managed_after: List defaults [] + property source_text: Text defaults "" +end + +create container OrmMigrationPlan: + property applied: List defaults [] + property pending: List defaults [] + property target_id: Text defaults "" +end + +define action called orm_migration_create_table with parameters table_spec: + create new OrmMigrationStep as migration_step: + operation is "create table" + after_table is table_spec + end + return migration_step +end action + +define action called orm_migration_drop_table with parameters table_spec: + create new OrmMigrationStep as migration_step: + operation is "drop table" + before_table is table_spec + end + return migration_step +end action + +define action called orm_migration_add_column with parameters before_table and after_table and field_spec: + create new OrmMigrationStep as migration_step: + operation is "add column" + before_table is before_table + after_table is after_table + column_spec is field_spec + end + return migration_step +end action + +define action called orm_migration_rebuild with parameters before_table and after_table and source_fields and target_fields: + create new OrmMigrationStep as migration_step: + operation is "rebuild table" + before_table is before_table + after_table is after_table + source_fields is source_fields + target_fields is target_fields + end + return migration_step +end action + +define action called orm_migration_sql with parameters sql_text and bound_values: + create new OrmMigrationStep as migration_step: + sql_text is sql_text + bound_values is bound_values + end + return migration_step +end action + +define action called orm_migration_create_index with parameters table_spec and index_spec: + create new OrmMigrationStep as migration_step: + operation is "create index" + after_table is table_spec + index_spec is index_spec + end + return migration_step +end action + +define action called orm_migration_drop_index with parameters table_spec and index_spec: + create new OrmMigrationStep as migration_step: + operation is "drop index" + before_table is table_spec + index_spec is index_spec + end + return migration_step +end action + +define action called orm_schema_signature with parameters table_spec: + check if table_spec is nothing: + return nothing + end check + store field_values as [] + for each field_spec in table_spec.model_spec.fields: + push with field_values and [field_spec.field_name, field_spec.value_type, field_spec.nullable, field_spec.primary_key, field_spec.generated, field_spec.unique_value, field_spec.sensitive, field_spec.has_default, field_spec.default_value, field_spec.server_default] + end for + store index_values as [] + for each index_spec in table_spec.model_spec.indexes: + push with index_values and [index_spec.index_name, index_spec.field_names, index_spec.unique_values] + end for + return [(orm_schema_create_sql of table_spec and table_spec.model_spec.table_name), field_values, index_values] +end action + +define action called orm_schema_signatures with parameters table_specs: + store signatures as [] + store table_names as [] + for each table_spec in table_specs: + store table_name as table_spec.model_spec.table_name + check if (tolowercase of table_name) starts with "_orm_" or (contains of table_names and (tolowercase of table_name)): + call orm_fail with "migration schema" and "managed table names must be unique and must not use the reserved _orm_ prefix" + end check + push with table_names and (tolowercase of table_name) + push with signatures and (orm_schema_signature of table_spec) + end for + return signatures +end action + +define action called orm_step_signatures with parameters migration_steps: + store signatures as [] + for each migration_step in migration_steps: + check if (contains of ["sql", "create table", "drop table", "add column", "rebuild table", "create index", "drop index"] and migration_step.operation) is equal to no: + call orm_fail with "migration step" and "unknown operation; use a typed schema helper or explicit parameterized SQL" + end check + check if migration_step.operation is equal to "sql" and (trim of migration_step.sql_text) is equal to "": + call orm_fail with "migration step" and "explicit SQL steps cannot be empty" + end check + store column_sql as "" + check if migration_step.column_spec is not nothing: + change column_sql to orm_schema_column of migration_step.column_spec + end check + store index_signature as nothing + check if migration_step.index_spec is not nothing: + store index_spec as migration_step.index_spec + change index_signature to [index_spec.index_name, index_spec.field_names, index_spec.unique_values] + end check + push with signatures and [migration_step.operation, (orm_schema_signature of migration_step.before_table), (orm_schema_signature of migration_step.after_table), migration_step.source_fields, migration_step.target_fields, column_sql, index_signature, migration_step.sql_text, migration_step.bound_values] + end for + return signatures +end action + +define action called orm_migration_checksum with parameters migration_spec: + // Ordered lists have stable JSON serialization; object/map key order does not. + // Only platform line endings are normalized. Comments and whitespace in the + // immutable source intentionally participate in its checksum. + store source_lf as replace "\r\n" with "\n" in migration_spec.source_text + store signature as ["orm-migration-v1", migration_spec.migration_id, migration_spec.migration_name, (orm_step_signatures of migration_spec.up_steps), (orm_step_signatures of migration_spec.down_steps), migration_spec.irreversible_reason, (orm_schema_signatures of migration_spec.managed_before), (orm_schema_signatures of migration_spec.managed_after), source_lf] + return sha256 of (stringify_json of signature) +end action + +define action called orm_migration_registry_check with parameters registry: + store known_ids as [] + store prior_after as nothing + store previous_id as "" + for each migration_spec in registry: + store id_text as migration_spec.migration_id + check if (length of id_text) is not equal to 14: + call orm_fail with "migration registry" and "use a unique increasing 14-digit UTC timestamp ID (YYYYMMDDhhmmss)" + end check + count from 0 to 13: + check if (contains of "0123456789" and (substring of id_text and count and 1)) is equal to no: + call orm_fail with "migration registry" and "migration IDs contain exactly 14 decimal digits" + end check + end count + check if contains of known_ids and id_text: + call orm_fail with "migration registry" and "duplicate migration ID; register each immutable version once" + end check + check if previous_id is not equal to "": + check if (orm_decimal_exceeds of id_text and previous_id) is equal to no: + call orm_fail with "migration registry" and "register migrations in increasing ID order" + end check + end check + store before_signature as stringify_json of (orm_schema_signatures of migration_spec.managed_before) + check if prior_after is not nothing and before_signature is not equal to prior_after: + call orm_fail with "migration registry" and "each migration's historical before schema must match the preceding after schema" + end check + check if (length of migration_spec.up_steps) is equal to 0: + call orm_fail with "migration registry" and "declare at least one up step; a scaffold is not an executable migration" + end check + for each direction_steps in [migration_spec.up_steps, migration_spec.down_steps]: + for each migration_step in direction_steps: + check if migration_step.operation is equal to "drop table" and migration_spec.irreversible_reason is equal to "": + call orm_fail with "migration registry" and "dropping a table requires an explicit irreversible reason; a recreated empty table does not restore records" + end check + check if migration_step.operation is equal to "rebuild table": + for each prior_field in migration_step.before_table.model_spec.fields: + check if (contains of migration_step.source_fields and prior_field.field_name) is equal to no and migration_spec.irreversible_reason is equal to "": + call orm_fail with "migration registry" and "discarding a column's values requires an explicit irreversible reason; adding an empty column back does not restore its values" + end check + end for + end check + end for + end for + store verified_checksum as orm_migration_checksum of migration_spec + change prior_after to stringify_json of (orm_schema_signatures of migration_spec.managed_after) + change previous_id to id_text + push with known_ids and id_text + end for + return known_ids +end action + +define action called orm_migration_ledger_sql: + return "CREATE TABLE _orm_migrations (position INTEGER PRIMARY KEY, migration_id TEXT UNIQUE NOT NULL, migration_name TEXT NOT NULL, checksum TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT (datetime('now')))" +end action + +define action called orm_migration_events_sql: + return "CREATE TABLE _orm_migration_events (event_id INTEGER PRIMARY KEY AUTOINCREMENT, migration_id TEXT NOT NULL, checksum TEXT NOT NULL, event_kind TEXT NOT NULL, recorded_at TEXT NOT NULL DEFAULT (datetime('now')))" +end action + +define action called orm_migration_ensure_history with parameters session: + check if (orm_schema_exists of session and "_orm_migrations") is equal to no: + store created_ledger as orm_sql_execute of session and orm_migration_ledger_sql and [] + store created_events as orm_sql_execute of session and orm_migration_events_sql and [] + end check + return yes +end action + +define action called orm_migration_expected_schema with parameters registry and applied_total: + check if (length of registry) is equal to 0: + return [] + end check + check if applied_total is equal to 0: + store first_migration as registry[0] + return first_migration.managed_before + end check + store last_migration as registry[applied_total minus 1] + return last_migration.managed_after +end action + +define action called orm_migration_schema_check with parameters session and registry and applied_total: + store expected_tables as orm_migration_expected_schema of registry and applied_total + store expected_names as [] + store expected_indexes as [] + for each table_spec in expected_tables: + push with expected_names and (tolowercase of table_spec.model_spec.table_name) + for each index_spec in table_spec.model_spec.indexes: + push with expected_indexes and (tolowercase of index_spec.index_name) + end for + end for + store checked_tables as orm_schema_require of session and expected_tables + store inspected_names as [] + store inspected_indexes as [] + for each migration_spec in registry: + for each historical_tables in [migration_spec.managed_before, migration_spec.managed_after]: + for each table_spec in historical_tables: + store table_name as table_spec.model_spec.table_name + store folded_name as tolowercase of table_name + check if (contains of expected_names and folded_name) is equal to no and (contains of inspected_names and folded_name) is equal to no: + check if orm_schema_exists of session and table_name: + call orm_fail with "schema drift" and ("unexpected managed table " with table_name with "; use an explicit supported legacy adoption or restore matching migration history") + end check + push with inspected_names and folded_name + end check + for each index_spec in table_spec.model_spec.indexes: + store index_name as index_spec.index_name + store folded_index as tolowercase of index_name + check if (contains of expected_indexes and folded_index) is equal to no and (contains of inspected_indexes and folded_index) is equal to no: + store index_rows as orm_sql_query of session and "SELECT name FROM sqlite_schema WHERE type='index' AND name=? COLLATE NOCASE" and [index_name] + check if (length of index_rows) is greater than 0: + call orm_fail with "schema drift" and ("unexpected managed index " with index_name with "; restore the index state declared by the applied migration version") + end check + push with inspected_indexes and folded_index + end check + end for + end for + end for + end for + return yes +end action + +define action called orm_migration_history with parameters session and registry: + store known_ids as orm_migration_registry_check of registry + store ledger_exists as orm_schema_exists of session and "_orm_migrations" + store events_exist as orm_schema_exists of session and "_orm_migration_events" + check if ledger_exists is not equal to events_exist: + call orm_fail with "migration history" and "one history table is missing; restore the matching database backup before proceeding" + end check + check if ledger_exists is equal to no: + return [] + end check + store ledger_schema as orm_sql_query of session and "SELECT name,sql FROM sqlite_schema WHERE type='table' AND name COLLATE NOCASE IN ('_orm_migrations','_orm_migration_events') ORDER BY name" and [] + for each schema_row in ledger_schema: + store expected_sql as orm_migration_ledger_sql + check if (tolowercase of schema_row["name"]) is equal to "_orm_migration_events": + change expected_sql to orm_migration_events_sql + end check + check if (orm_schema_normalize of schema_row["sql"]) is not equal to (orm_schema_normalize of expected_sql): + call orm_fail with "migration history" and "history table structure changed; restore matching history before proceeding" + end check + end for + store applied_rows as orm_sql_query of session and "SELECT position,migration_id,migration_name,checksum FROM _orm_migrations ORDER BY position" and [] + check if (length of applied_rows) is greater than (length of registry): + call orm_fail with "migration history" and "an applied migration file is missing; restore the immutable registered version" + end check + store applied_ids as [] + store row_position as 0 + for each applied_row in applied_rows: + store migration_spec as registry[row_position] + check if applied_row["position"] is not equal to (row_position plus 1) or applied_row["migration_id"] is not equal to migration_spec.migration_id: + call orm_fail with "migration history" and "applied history is not a contiguous registered prefix; restore missing versions or matching history" + end check + check if applied_row["checksum"] is not equal to (orm_migration_checksum of migration_spec) or applied_row["migration_name"] is not equal to migration_spec.migration_name: + call orm_fail with "migration checksum" and "an applied migration changed; restore its original immutable source and author a new migration" + end check + push with applied_ids and migration_spec.migration_id + change row_position to row_position plus 1 + end for + store event_rows as orm_sql_query of session and "SELECT migration_id,checksum,event_kind FROM _orm_migration_events ORDER BY event_id" and [] + store replayed_ids as [] + for each event_row in event_rows: + store event_id as event_row["migration_id"] + check if (contains of known_ids and event_id) is equal to no: + call orm_fail with "migration history" and "an event references a missing migration file; restore the complete immutable registry" + end check + store registered_position as 0 + for each known_id in known_ids: + check if known_id is equal to event_id: + break + end check + change registered_position to registered_position plus 1 + end for + store migration_spec as registry[registered_position] + check if event_row["checksum"] is not equal to (orm_migration_checksum of migration_spec): + call orm_fail with "migration checksum" and "a previously applied or rolled-back migration changed; restore its original immutable source" + end check + store event_kind as event_row["event_kind"] + check if event_kind is equal to "apply" or event_kind is equal to "adopt": + check if registered_position is not equal to (length of replayed_ids): + call orm_fail with "migration history" and "migration event order is inconsistent; restore matching history" + end check + push with replayed_ids and event_id + otherwise: + check if event_kind is not equal to "rollback" or (length of replayed_ids) is equal to 0: + call orm_fail with "migration history" and "unknown or unmatched rollback event; restore matching history" + end check + store prior_id as pop of replayed_ids + check if prior_id is not equal to event_id: + call orm_fail with "migration history" and "rollback events are out of order; restore matching history" + end check + end check + end for + check if (stringify_json of replayed_ids) is not equal to (stringify_json of applied_ids): + call orm_fail with "migration history" and "applied ledger and event history disagree; restore their matching backup" + end check + return applied_rows +end action + +define action called orm_migration_target with parameters registry and target_id: + check if target_id is equal to "latest": + return length of registry + end check + check if target_id is equal to "zero": + return 0 + end check + store migration_position as 0 + for each migration_spec in registry: + change migration_position to migration_position plus 1 + check if migration_spec.migration_id is equal to target_id: + return migration_position + end check + end for + call orm_fail with "migration target" and "target is not registered; use an exact migration ID, latest, or zero" +end action + +define action called orm_migration_plan with parameters session and registry and target_id: + store conn as session.connection + // All reads use one SQLite snapshot, so a concurrent commit cannot pair an + // old ledger with a new schema and produce a false corruption diagnosis. + in transaction on conn: + store applied_rows as orm_migration_history of session and registry + store checked_schema as orm_migration_schema_check of session and registry and (length of applied_rows) + store target_position as orm_migration_target of registry and target_id + store pending_ids as [] + store migration_position as 0 + for each migration_spec in registry: + change migration_position to migration_position plus 1 + check if migration_position is greater than (length of applied_rows) and migration_position is less than or equal to target_position: + push with pending_ids and migration_spec.migration_id + end check + end for + create new OrmMigrationPlan as migration_plan: + applied is applied_rows + pending is pending_ids + target_id is target_id + end + return migration_plan + end transaction +end action + +define action called orm_migration_status with parameters session and registry: + return orm_migration_plan of session and registry and "latest" +end action + +define action called orm_migration_run_step with parameters session and migration_step: + store operation as migration_step.operation + check if operation is equal to "create table": + return orm_schema_create of session and migration_step.after_table + end check + check if operation is equal to "drop table": + return orm_sql_execute of session and ("DROP TABLE " with (orm_quote of migration_step.before_table.model_spec.table_name)) and [] + end check + check if operation is equal to "add column": + return orm_sql_execute of session and ("ALTER TABLE " with (orm_quote of migration_step.before_table.model_spec.table_name) with " ADD COLUMN " with (orm_schema_column of migration_step.column_spec)) and [] + end check + check if operation is equal to "create index": + return orm_sql_execute of session and (orm_schema_index_sql of migration_step.after_table and migration_step.index_spec) and [] + end check + check if operation is equal to "drop index": + return orm_sql_execute of session and ("DROP INDEX " with (orm_quote of migration_step.index_spec.index_name)) and [] + end check + check if operation is equal to "rebuild table": + return orm_schema_rebuild of session and migration_step.before_table and migration_step.after_table and migration_step.source_fields and migration_step.target_fields + end check + check if operation is equal to "sql": + return orm_sql_execute of session and migration_step.sql_text and migration_step.bound_values + end check + call orm_fail with "migration step" and "unknown step; use a typed schema helper or explicit parameterized SQL" +end action + +define action called orm_migration_record with parameters session and migration_spec and migration_position and event_kind: + store checksum_text as orm_migration_checksum of migration_spec + store recorded_migration as orm_sql_execute of session and "INSERT INTO _orm_migrations(position,migration_id,migration_name,checksum) VALUES (?,?,?,?)" and [migration_position, migration_spec.migration_id, migration_spec.migration_name, checksum_text] + store recorded_event as orm_sql_execute of session and "INSERT INTO _orm_migration_events(migration_id,checksum,event_kind) VALUES (?,?,?)" and [migration_spec.migration_id, checksum_text, event_kind] + return yes +end action + +define action called orm_migrate with parameters session and registry and target_id: + store checked_plan as orm_migration_plan of session and registry and target_id + store target_position as orm_migration_target of registry and target_id + check if target_position is less than (length of checked_plan.applied): + call orm_fail with "migration target" and "up cannot roll back; use rollback_to or rollback_count explicitly" + end check + store conn as session.connection + store expected_floor as length of checked_plan.applied + repeat while yes: + store finished as no + in transaction on conn for schema changes: + // Another process may have progressed while this caller waited for + // the bounded native write lock. Re-read everything under that lock. + store applied_rows as orm_migration_history of session and registry + store checked_schema as orm_migration_schema_check of session and registry and (length of applied_rows) + check if (length of applied_rows) is greater than target_position or (length of applied_rows) is less than expected_floor: + call orm_fail with "migration concurrency" and "another runner moved history beyond this target or in the opposite direction; inspect status and retry the intended target" + end check + check if (length of applied_rows) is equal to target_position: + change finished to yes + otherwise: + store migration_spec as registry[(length of applied_rows)] + store history_ready as orm_migration_ensure_history of session + for each migration_step in migration_spec.up_steps: + store step_result as orm_migration_run_step of session and migration_step + end for + store verified_schema as orm_migration_schema_check of session and registry and ((length of applied_rows) plus 1) + store recorded_migration as orm_migration_record of session and migration_spec and ((length of applied_rows) plus 1) and "apply" + change expected_floor to (length of applied_rows) plus 1 + end check + end transaction + check if finished: + break + end check + end repeat + return orm_migration_status of session and registry +end action + +define action called orm_rollback_to with parameters session and registry and target_id: + store current_plan as orm_migration_status of session and registry + store target_position as orm_migration_target of registry and target_id + check if target_position is greater than (length of current_plan.applied): + call orm_fail with "rollback target" and "target is newer than the current version; use migrate explicitly" + end check + // Preflight the complete requested range before making even one change. + store migration_position as 0 + for each migration_spec in registry: + change migration_position to migration_position plus 1 + check if migration_position is greater than target_position and migration_position is less than or equal to (length of current_plan.applied): + check if migration_spec.irreversible_reason is not equal to "" or (length of migration_spec.down_steps) is equal to 0: + call orm_fail with "irreversible migration" and (migration_spec.migration_id with ": " with migration_spec.irreversible_reason with "; restore a matching database and uploads backup instead") + end check + end check + end for + store conn as session.connection + store expected_ceiling as length of current_plan.applied + repeat while yes: + store finished as no + in transaction on conn for schema changes: + store applied_rows as orm_migration_history of session and registry + store verified_before as orm_migration_schema_check of session and registry and (length of applied_rows) + check if (length of applied_rows) is less than target_position or (length of applied_rows) is greater than expected_ceiling: + call orm_fail with "migration concurrency" and "another runner moved history beyond this target or in the opposite direction; inspect status and retry the intended target" + end check + check if (length of applied_rows) is equal to target_position: + change finished to yes + otherwise: + store migration_spec as registry[(length of applied_rows) minus 1] + check if migration_spec.irreversible_reason is not equal to "" or (length of migration_spec.down_steps) is equal to 0: + call orm_fail with "irreversible migration" and "history changed while waiting for the lock; review rollback status again" + end check + for each migration_step in migration_spec.down_steps: + store step_result as orm_migration_run_step of session and migration_step + end for + store verified_after as orm_migration_schema_check of session and registry and ((length of applied_rows) minus 1) + store removed_migration as orm_sql_execute of session and "DELETE FROM _orm_migrations WHERE migration_id=?" and [migration_spec.migration_id] + store recorded_event as orm_sql_execute of session and "INSERT INTO _orm_migration_events(migration_id,checksum,event_kind) VALUES (?,?,'rollback')" and [migration_spec.migration_id, (orm_migration_checksum of migration_spec)] + change expected_ceiling to (length of applied_rows) minus 1 + end check + end transaction + check if finished: + break + end check + end repeat + return orm_migration_status of session and registry +end action + +define action called orm_rollback_count with parameters session and registry and rollback_total: + store current_plan as orm_migration_status of session and registry + check if (typeof of rollback_total) is not equal to "Number" or rollback_total is less than 1 or rollback_total is greater than (length of current_plan.applied) or (floor of rollback_total) is not equal to rollback_total: + call orm_fail with "rollback count" and "use a positive whole count no larger than the applied migration count" + end check + store target_position as (length of current_plan.applied) minus rollback_total + store target_id as "zero" + check if target_position is greater than 0: + store target_migration as registry[target_position minus 1] + change target_id to target_migration.migration_id + end check + return orm_rollback_to of session and registry and target_id +end action diff --git a/lib/orm/models.wfl b/lib/orm/models.wfl new file mode 100644 index 0000000..05195c7 --- /dev/null +++ b/lib/orm/models.wfl @@ -0,0 +1,121 @@ +include from "types.wfl" + +create container OrmModel: + property table_name: Text defaults "" + property fields: List defaults [] + property indexes: List defaults [] + property relationships: List defaults [] + + action relate needs relationship_spec: OrmRelationship: + // Replace the list instead of mutating a caller-owned/default list. + store revised_relations as [] + for each existing_relation in relationships: + push with revised_relations and existing_relation + end for + push with revised_relations and relationship_spec + change relationships to revised_relations + end +end + +define action called orm_field_named with parameters model_spec and requested_name: + for each field_spec in model_spec.fields: + check if field_spec.field_name is equal to requested_name: + return field_spec + end check + end for + return nothing +end action + +define action called orm_primary_field with parameters model_spec: + for each field_spec in model_spec.fields: + check if field_spec.primary_key: + return field_spec + end check + end for + return nothing +end action + +define action called orm_model_problem with parameters model_spec: + store name_problem as orm_identifier_problem of model_spec.table_name + check if name_problem is not equal to "": + return "table: " with name_problem + end check + check if (length of model_spec.fields) is equal to 0: + return "declare at least one field and exactly one primary key" + end check + store primary_total as 0 + store field_names as [] + for each field_spec in model_spec.fields: + store field_problem as orm_field_problem of field_spec + check if field_problem is not equal to "": + return "field declaration: " with field_problem + end check + store folded_name as tolowercase of field_spec.field_name + check if contains of field_names and folded_name: + return "field names must be distinct without regard to ASCII case" + end check + push with field_names and folded_name + check if field_spec.primary_key: + change primary_total to primary_total plus 1 + end check + end for + check if primary_total is not equal to 1: + return "declare exactly one primary key; composite primary keys are not supported" + end check + store index_names as [] + for each index_spec in model_spec.indexes: + store index_problem as orm_identifier_problem of index_spec.index_name + check if index_problem is not equal to "": + return "index: " with index_problem + end check + store folded_index as tolowercase of index_spec.index_name + check if contains of index_names and folded_index: + return "index names must be distinct without regard to ASCII case" + end check + push with index_names and folded_index + check if (length of index_spec.field_names) is equal to 0: + return "declare at least one field in each index" + end check + store indexed_fields as [] + for each indexed_name in index_spec.field_names: + check if (orm_field_named of model_spec and indexed_name) is nothing: + return "an index refers to an undeclared field" + end check + check if contains of indexed_fields and indexed_name: + return "an index cannot repeat a field" + end check + push with indexed_fields and indexed_name + end for + end for + store relation_names as [] + for each relation_spec in model_spec.relationships: + store relation_problem as orm_identifier_problem of relation_spec.relation_name + check if relation_problem is not equal to "": + return "relationship: " with relation_problem + end check + check if contains of relation_names and relation_spec.relation_name: + return "relationship names must be distinct" + end check + push with relation_names and relation_spec.relation_name + store local_spec as orm_field_named of model_spec and relation_spec.local_field + check if local_spec is nothing: + return "a relationship refers to an undeclared local field" + end check + check if relation_spec.related_model is nothing: + return "a relationship must name its related model" + end check + store related_spec as orm_field_named of relation_spec.related_model and relation_spec.related_field + check if related_spec is nothing: + return "a relationship refers to an undeclared related field" + end check + check if local_spec.value_type is not equal to related_spec.value_type: + return "relationship fields must use the same value type" + end check + check if relation_spec.many is equal to no: + check if related_spec.primary_key is equal to no and related_spec.unique_value is equal to no: + return "a single-record relationship must target a primary or unique field" + end check + end check + end for + return "" +end action diff --git a/lib/orm/predicates.wfl b/lib/orm/predicates.wfl new file mode 100644 index 0000000..30846e9 --- /dev/null +++ b/lib/orm/predicates.wfl @@ -0,0 +1,225 @@ +include from "records.wfl" + +create container OrmPredicate: + property predicate_kind: Text defaults "comparison" + property field_name: Text defaults "" + property comparison: Text defaults "equal" + property compared_value defaults nothing + property children: List defaults [] +end + +create container OrmSql: + property sql_text: Text defaults "" + property bound_values: List defaults [] + // true/false identify structural constants; unknown depends on stored rows. + property constant_kind: Text defaults "unknown" +end + +define action called orm_compare with parameters field_name and comparison and compared_value: + create new OrmPredicate as predicate: + field_name is field_name + comparison is comparison + compared_value is compared_value + end + return predicate +end action + +define action called orm_all with parameters predicates: + create new OrmPredicate as predicate: + predicate_kind is "all" + children is predicates + end + return predicate +end action + +define action called orm_any with parameters predicates: + create new OrmPredicate as predicate: + predicate_kind is "any" + children is predicates + end + return predicate +end action + +define action called orm_not with parameters predicate: + create new OrmPredicate as negation: + predicate_kind is "not" + children is [predicate] + end + return negation +end action + +define action called orm_quote with parameters identifier: + store problem_text as orm_identifier_problem of identifier + check if problem_text is not equal to "": + call orm_fail with "identifier" and problem_text + end check + return "\"" with identifier with "\"" +end action + +define action called orm_sql with parameters sql_text and bound_values and constant_kind: + create new OrmSql as compiled: + sql_text is sql_text + bound_values is bound_values + constant_kind is constant_kind + end + return compiled +end action + +define action called orm_compile_predicate with parameters model_spec and predicate: + check if predicate is nothing: + return orm_sql of "1" and [] and "true" + end check + check if predicate.predicate_kind is equal to "all" or predicate.predicate_kind is equal to "any": + store parts as [] + store bound_values as [] + store glue as " AND " + store constant_kind as "true" + store decisive as "false" + check if predicate.predicate_kind is equal to "any": + change glue to " OR " + change constant_kind to "false" + change decisive to "true" + end check + store has_unknown as no + store has_decisive as no + for each child_predicate in predicate.children: + store child_sql as orm_compile_predicate of model_spec and child_predicate + push with parts and child_sql.sql_text + for each bound_value in child_sql.bound_values: + push with bound_values and bound_value + end for + check if child_sql.constant_kind is equal to "unknown": + change has_unknown to yes + end check + check if child_sql.constant_kind is equal to decisive: + change has_decisive to yes + end check + end for + check if has_unknown: + change constant_kind to "unknown" + end check + check if has_decisive: + change constant_kind to decisive + end check + check if (length of parts) is equal to 0: + check if constant_kind is equal to "true": + return orm_sql of "1" and [] and "true" + end check + return orm_sql of "0" and [] and "false" + end check + return orm_sql of ("(" with (join of parts and glue) with ")") and bound_values and constant_kind + end check + check if predicate.predicate_kind is equal to "not": + check if (length of predicate.children) is not equal to 1: + call orm_fail with "filter" and "a negation requires exactly one predicate" + end check + store child_sql as orm_compile_predicate of model_spec and predicate.children[0] + store constant_kind as "unknown" + check if child_sql.constant_kind is equal to "true": + change constant_kind to "false" + end check + check if child_sql.constant_kind is equal to "false": + change constant_kind to "true" + end check + return orm_sql of ("(NOT " with child_sql.sql_text with ")") and child_sql.bound_values and constant_kind + end check + check if predicate.predicate_kind is not equal to "comparison": + call orm_fail with "filter" and "use a comparison, all, any, or not predicate" + end check + store field_spec as orm_require_field of model_spec and predicate.field_name + store column_sql as orm_quote of field_spec.field_name + store comparison as predicate.comparison + store compared_value as predicate.compared_value + check if comparison is equal to "in": + check if (typeof of compared_value) is not equal to "List": + call orm_fail with "filter" and "in requires a list of values" + end check + check if (length of compared_value) is greater than 500: + call orm_fail with "filter" and "in accepts at most 500 values; split larger work into bounded batches" + end check + store placeholders as [] + store bound_values as [] + store has_null as no + for each candidate_value in compared_value: + check if isnothing of candidate_value: + change has_null to yes + otherwise: + store value_problem as orm_value_problem of field_spec and candidate_value + check if value_problem is not equal to "": + call orm_fail with "filter" and value_problem + end check + push with placeholders and "?" + push with bound_values and candidate_value + end check + end for + store sql_text as "0" + store constant_kind as "false" + check if (length of placeholders) is greater than 0: + change sql_text to column_sql with " IN (" with (join of placeholders and ", ") with ")" + change constant_kind to "unknown" + end check + check if has_null: + change sql_text to "(" with sql_text with " OR " with column_sql with " IS NULL)" + change constant_kind to "unknown" + end check + return orm_sql of sql_text and bound_values and constant_kind + end check + check if comparison is equal to "is null" or comparison is equal to "is not null": + check if (isnothing of compared_value) is equal to no: + call orm_fail with "filter" and "is null and is not null take nothing as their comparison value" + end check + check if comparison is equal to "is null": + return orm_sql of (column_sql with " IS NULL") and [] and "unknown" + end check + return orm_sql of (column_sql with " IS NOT NULL") and [] and "unknown" + end check + check if isnothing of compared_value: + check if comparison is equal to "equal": + return orm_sql of (column_sql with " IS NULL") and [] and "unknown" + end check + check if comparison is equal to "not equal": + return orm_sql of (column_sql with " IS NOT NULL") and [] and "unknown" + end check + call orm_fail with "filter" and "null supports only equality, inequality, membership, or explicit null predicates" + end check + store value_problem as orm_value_problem of field_spec and compared_value + check if value_problem is not equal to "": + call orm_fail with "filter" and value_problem + end check + store operator_sql as "" + check if comparison is equal to "equal": + change operator_sql to " = " + end check + check if comparison is equal to "not equal": + change operator_sql to " <> " + end check + check if comparison is equal to "less than": + change operator_sql to " < " + end check + check if comparison is equal to "less or equal": + change operator_sql to " <= " + end check + check if comparison is equal to "greater than": + change operator_sql to " > " + end check + check if comparison is equal to "greater or equal": + change operator_sql to " >= " + end check + check if operator_sql is not equal to "": + return orm_sql of (column_sql with operator_sql with "?") and [compared_value] and "unknown" + end check + check if comparison is equal to "contains" or comparison is equal to "starts with": + check if field_spec.value_type is not equal to "text": + call orm_fail with "filter" and "contains and starts with require a text field" + end check + store pattern_text as replace "\\" with "\\\\" in compared_value + change pattern_text to replace "%" with "\\%" in pattern_text + change pattern_text to replace "_" with "\\_" in pattern_text + change pattern_text to pattern_text with "%" + check if comparison is equal to "contains": + change pattern_text to "%" with pattern_text + end check + return orm_sql of (column_sql with " LIKE ? ESCAPE '\\'") and [pattern_text] and "unknown" + end check + call orm_fail with "filter" and "unsupported comparison; use the documented comparison names" +end action diff --git a/lib/orm/queries.wfl b/lib/orm/queries.wfl new file mode 100644 index 0000000..bfc52f2 --- /dev/null +++ b/lib/orm/queries.wfl @@ -0,0 +1,237 @@ +include from "predicates.wfl" + +create container OrmOrder: + property field_name: Text defaults "" + property direction: Text defaults "ascending" +end + +create container OrmQuery: + property model_spec: OrmModel + property predicate defaults nothing + property ordering: List defaults [] + property selected_fields: List defaults [] + property page_limit: Number defaults 100 + property page_offset: Number defaults 0 + property whole_table: Boolean defaults no + + action set_predicate needs new_predicate: OrmPredicate: + change predicate to new_predicate + end + action set_ordering needs new_ordering: List: + change ordering to new_ordering + end + action set_projection needs new_projection: List: + change selected_fields to new_projection + end + action set_page needs new_limit: Number, new_offset: Number: + change page_limit to new_limit + change page_offset to new_offset + end + action allow_whole_table: + change whole_table to yes + end +end + +define action called orm_query with parameters model_spec: + store valid as orm_require_model of model_spec + create new OrmQuery as requested: + model_spec is model_spec + ordering is [] + selected_fields is [] + end + return requested +end action + +define action called orm_where with parameters requested and predicate: + // Compile before changing a query so a rejected predicate is atomic. + store valid as orm_compile_predicate of requested.model_spec and predicate + requested.set_predicate(predicate) + return requested +end action + +define action called orm_order_by with parameters requested and field_name and direction: + store field_spec as orm_require_field of requested.model_spec and field_name + check if direction is not equal to "ascending" and direction is not equal to "descending": + call orm_fail with "order" and "direction must be ascending or descending" + end check + create new OrmOrder as order_spec: + field_name is field_name + direction is direction + end + store revised_ordering as [] + for each existing_order in requested.ordering: + check if existing_order.field_name is equal to field_name: + call orm_fail with "order" and "a field cannot appear twice in the ordering" + end check + push with revised_ordering and existing_order + end for + push with revised_ordering and order_spec + requested.set_ordering(revised_ordering) + return requested +end action + +define action called orm_select with parameters requested and field_names: + store seen_fields as [] + for each field_name in field_names: + store declared as orm_require_field of requested.model_spec and field_name + check if contains of seen_fields and field_name: + call orm_fail with "projection" and "a field cannot appear twice in the projection" + end check + push with seen_fields and field_name + end for + requested.set_projection(seen_fields) + return requested +end action + +define action called orm_page with parameters requested and page_limit and page_offset: + check if (typeof of page_limit) is not equal to "Number" or (typeof of page_offset) is not equal to "Number": + call orm_fail with "page" and "limit and offset must be whole numbers" + end check + check if page_limit is less than 1 or page_limit is greater than 1000 or (floor of page_limit) is not equal to page_limit: + call orm_fail with "page" and "limit must be a whole number from 1 to 1000" + end check + check if page_offset is less than 0 or page_offset is greater than 9007199254740991 or (floor of page_offset) is not equal to page_offset: + call orm_fail with "page" and "offset must be a nonnegative whole number in the exact numeric range" + end check + requested.set_page(page_limit, page_offset) + return requested +end action + +// Deliberate whole-table maintenance is explicit at the call site. +define action called orm_whole_table with parameters requested: + requested.allow_whole_table() + return requested +end action + +define action called orm_projection_sql with parameters model_spec and field_names: + store projected as [] + for each field_spec in model_spec.fields: + check if (length of field_names) is equal to 0 or (contains of field_names and field_spec.field_name): + store quoted_name as orm_quote of field_spec.field_name + store projected_sql as quoted_name + check if field_spec.value_type is equal to "identity": + change projected_sql to "CAST(" with quoted_name with " AS TEXT) AS " with quoted_name + end check + push with projected and projected_sql + end check + end for + return join of projected and ", " +end action + +// SQLite offers three aliases for its physical row identity. Choose one that +// does not shadow a declared column; nullable logical keys need this identity +// to distinguish multiple NULL-key rows in stable pages and bulk writes. +define action called orm_rowid_name with parameters model_spec: + for each candidate_name in ["_rowid_" and "rowid" and "oid"]: + store available_name as yes + for each field_spec in model_spec.fields: + check if (to_lowercase of field_spec.field_name) is equal to candidate_name: + change available_name to no + end check + end for + check if available_name: + return candidate_name + end check + end for + call orm_fail with "row identity" and "all SQLite rowid aliases are shadowed; leave one of _rowid_, rowid, or oid undeclared" +end action + +define action called orm_compile_select with parameters requested and result_kind: + store valid as orm_require_model of requested.model_spec + store page_valid as orm_page of requested and requested.page_limit and requested.page_offset + store projection_valid as orm_select of requested and requested.selected_fields + store filter_sql as orm_compile_predicate of requested.model_spec and requested.predicate + store projection_sql as orm_projection_sql of requested.model_spec and requested.selected_fields + check if result_kind is equal to "count": + change projection_sql to "COUNT(*) AS orm_total" + otherwise: + check if result_kind is equal to "exists": + change projection_sql to "1 AS orm_exists" + otherwise: + check if result_kind is equal to "row identity": + change projection_sql to (orm_quote of requested.model_spec.table_name) with "." with (orm_quote of (orm_rowid_name of requested.model_spec)) + otherwise: + check if result_kind is not equal to "records": + call orm_fail with "query" and "unknown result kind" + end check + end check + end check + end check + store sql_text as "SELECT " with projection_sql with " FROM " with (orm_quote of requested.model_spec.table_name) with " WHERE " with filter_sql.sql_text + store bound_values as [] + for each bound_value in filter_sql.bound_values: + push with bound_values and bound_value + end for + check if result_kind is equal to "records" or result_kind is equal to "row identity": + store ordering_parts as [] + store ordered_fields as [] + for each order_spec in requested.ordering: + store field_spec as orm_require_field of requested.model_spec and order_spec.field_name + check if order_spec.direction is not equal to "ascending" and order_spec.direction is not equal to "descending": + call orm_fail with "order" and "direction must be ascending or descending" + end check + check if contains of ordered_fields and order_spec.field_name: + call orm_fail with "order" and "a field cannot appear twice in the ordering" + end check + push with ordered_fields and order_spec.field_name + store suffix as " ASC" + check if order_spec.direction is equal to "descending": + change suffix to " DESC" + end check + // Qualify the stored column: an exact-text projection may use the + // same alias, and SQLite resolves an unqualified ORDER BY to it. + push with ordering_parts and ((orm_quote of requested.model_spec.table_name) with "." with (orm_quote of order_spec.field_name) with suffix) + end for + store key_spec as orm_primary_field of requested.model_spec + check if (contains of ordered_fields and key_spec.field_name) is equal to no: + push with ordering_parts and ((orm_quote of requested.model_spec.table_name) with "." with (orm_quote of key_spec.field_name) with " ASC") + end check + check if key_spec.nullable: + push with ordering_parts and ((orm_quote of requested.model_spec.table_name) with "." with (orm_quote of (orm_rowid_name of requested.model_spec)) with " ASC") + end check + change sql_text to sql_text with " ORDER BY " with (join of ordering_parts and ", ") with " LIMIT ? OFFSET ?" + push with bound_values and requested.page_limit + push with bound_values and requested.page_offset + end check + check if result_kind is equal to "exists": + change sql_text to sql_text with " LIMIT 1" + end check + return orm_sql of sql_text and bound_values and filter_sql.constant_kind +end action + +define action called orm_map_row with parameters model_spec and native_row: + store assignments as [] + for each field_spec in model_spec.fields: + check if native_row contains field_spec.field_name: + store field_value as native_row[field_spec.field_name] + check if field_spec.value_type is equal to "boolean" and (isnothing of field_value) is equal to no: + check if field_value is equal to 1: + change field_value to yes + otherwise: + check if field_value is equal to 0: + change field_value to no + end check + end check + end check + store problem_text as orm_value_problem of field_spec and field_value + check if problem_text is not equal to "": + call orm_fail with "map stored row" and ("field " with field_spec.field_name with ": " with problem_text with "; inspect the schema and repair invalid stored data") + end check + push with assignments and (orm_value of field_spec.field_name and field_value) + end check + end for + store key_spec as orm_primary_field of model_spec + store identity_value as nothing + check if native_row contains key_spec.field_name: + change identity_value to native_row[key_spec.field_name] + end check + create new OrmRecord as mapped: + model_spec is model_spec + assigned_values is assignments + loaded_relations is [] + persisted is yes + original_key is identity_value + source_row is native_row + end + return mapped +end action diff --git a/lib/orm/records.wfl b/lib/orm/records.wfl new file mode 100644 index 0000000..e38fa9c --- /dev/null +++ b/lib/orm/records.wfl @@ -0,0 +1,187 @@ +include from "models.wfl" + +// Only validated identifiers appear in diagnostics, never assigned values. +define action called orm_fail with parameters operation and guidance: + call raise_error with ("ORM " with operation with ": " with guidance) +end action + +define action called orm_require_model with parameters model_spec: + store model_problem as orm_model_problem of model_spec + check if model_problem is not equal to "": + call orm_fail with "model" and model_problem + end check + return yes +end action + +define action called orm_require_field with parameters model_spec and field_name: + store field_spec as orm_field_named of model_spec and field_name + check if field_spec is nothing: + call orm_fail with "field" and "unknown field; use a field declared by this model" + end check + return field_spec +end action + +create container OrmAssignment: + property field_name: Text defaults "" + property field_value defaults nothing +end + +// Assignment lists are explicit data, so every key can be checked before any +// change. No input map is silently filtered to a list of known fields. +define action called orm_value with parameters field_name and field_value: + create new OrmAssignment as assignment: + field_name is field_name + field_value is field_value + end + return assignment +end action + +create container OrmRecord: + property model_spec: OrmModel + property assigned_values: List defaults [] + property loaded_relations: List defaults [] + property persisted: Boolean defaults no + property original_key defaults nothing + // Read-only native projection for application compatibility adapters. + property source_row defaults nothing + // Physical SQLite row identity returned by an INSERT, kept as exact text. + property inserted_rowid defaults nothing + + action remember_insert needs rowid_value: Text: + change inserted_rowid to rowid_value + end + + action replace_values needs replacement: List: + change assigned_values to replacement + end + + action mark_persisted needs identity_value: OrmAssignment: + change original_key to identity_value.field_value + change persisted to yes + end + + action replace_relations needs replacement: List: + change loaded_relations to replacement + end +end + +define action called orm_record with parameters model_spec: + store valid as orm_require_model of model_spec + create new OrmRecord as fresh_record: + model_spec is model_spec + assigned_values is [] + loaded_relations is [] + end + return fresh_record +end action + +define action called orm_has with parameters record_value and field_name: + store declared as orm_require_field of record_value.model_spec and field_name + for each assignment in record_value.assigned_values: + check if assignment.field_name is equal to field_name: + return yes + end check + end for + return no +end action + +define action called orm_get with parameters record_value and field_name: + store declared as orm_require_field of record_value.model_spec and field_name + for each assignment in record_value.assigned_values: + check if assignment.field_name is equal to field_name: + return assignment.field_value + end check + end for + call orm_fail with "read" and "field is missing; assign it or include it in the query projection (use orm_has to check presence)" +end action + +define action called orm_assignment_problem with parameters model_spec and assignment and trusted_sensitive: + store field_spec as orm_field_named of model_spec and assignment.field_name + check if field_spec is nothing: + return "unknown field; use only declared fields" + end check + check if field_spec.sensitive and trusted_sensitive is equal to no: + return "sensitive fields require explicit orm_set_sensitive; do not assign untrusted form or JSON values to them" + end check + store value_problem as orm_value_problem of field_spec and assignment.field_value + check if value_problem is not equal to "": + return "field " with field_spec.field_name with ": " with value_problem + end check + return "" +end action + +// Internal update is copy-on-write; both ordinary and explicit sensitive setters +// use it. The caller validates the entire batch before invoking it. +define action called orm_replace_assignment with parameters record_value and replacement: + store revised_values as [] + for each assignment in record_value.assigned_values: + check if assignment.field_name is not equal to replacement.field_name: + push with revised_values and assignment + end check + end for + push with revised_values and replacement + record_value.replace_values(revised_values) + return record_value +end action + +define action called orm_assign with parameters record_value and assignments: + store valid as orm_require_model of record_value.model_spec + store assigned_names as [] + for each assignment in assignments: + store assignment_problem as orm_assignment_problem of record_value.model_spec and assignment and no + check if assignment_problem is not equal to "": + call orm_fail with "assign" and assignment_problem + end check + check if contains of assigned_names and assignment.field_name: + call orm_fail with "assign" and "a batch cannot repeat a field" + end check + push with assigned_names and assignment.field_name + end for + for each assignment in assignments: + store changed_record as orm_replace_assignment of record_value and assignment + end for + return record_value +end action + +define action called orm_set with parameters record_value and field_name and field_value: + store assignment as orm_value of field_name and field_value + return orm_assign of record_value and [assignment] +end action + +define action called orm_set_sensitive with parameters record_value and field_name and field_value: + store valid as orm_require_model of record_value.model_spec + store assignment as orm_value of field_name and field_value + store assignment_problem as orm_assignment_problem of record_value.model_spec and assignment and yes + check if assignment_problem is not equal to "": + call orm_fail with "assign sensitive field" and assignment_problem + end check + return orm_replace_assignment of record_value and assignment +end action + +define action called orm_record_problem with parameters record_value and for_insert: + store model_problem as orm_model_problem of record_value.model_spec + check if model_problem is not equal to "": + return model_problem + end check + store assigned_names as [] + for each assignment in record_value.assigned_values: + store value_problem as orm_assignment_problem of record_value.model_spec and assignment and yes + check if value_problem is not equal to "": + return value_problem + end check + check if contains of assigned_names and assignment.field_name: + return "a record cannot repeat a field" + end check + push with assigned_names and assignment.field_name + end for + check if for_insert: + for each field_spec in record_value.model_spec.fields: + check if (contains of assigned_names and field_spec.field_name) is equal to no: + check if field_spec.nullable is equal to no and field_spec.generated is equal to no and field_spec.has_default is equal to no and field_spec.server_default is equal to "": + return "missing required field " with field_spec.field_name with "; supply it before saving" + end check + end check + end for + end check + return "" +end action diff --git a/lib/orm/relationships.wfl b/lib/orm/relationships.wfl new file mode 100644 index 0000000..4836be5 --- /dev/null +++ b/lib/orm/relationships.wfl @@ -0,0 +1,120 @@ +include from "writes.wfl" + +create container OrmLoadedRelation: + property relation_name: Text defaults "" + property related_records: List defaults [] + property many: Boolean defaults no +end + +define action called orm_relationship_named with parameters model_spec and relation_name: + for each relation_spec in model_spec.relationships: + check if relation_spec.relation_name is equal to relation_name: + return relation_spec + end check + end for + call orm_fail with "relationship" and "unknown relationship; declare it on the model before loading it" +end action + +define action called orm_related with parameters record_value and relation_name: + store declared as orm_relationship_named of record_value.model_spec and relation_name + for each loaded in record_value.loaded_relations: + check if loaded.relation_name is equal to relation_name: + check if loaded.many: + return loaded.related_records + end check + check if (length of loaded.related_records) is greater than 0: + return loaded.related_records[0] + end check + return nothing + end check + end for + call orm_fail with "relationship" and "relationship is not loaded; call orm_load explicitly (no query is performed by orm_related)" +end action + +define action called orm_load with parameters session and source_records and relation_name: + check if (length of source_records) is equal to 0: + return source_records + end check + check if (length of source_records) is greater than 1000: + call orm_fail with "relationship" and "load relationships for at most 1000 source records at a time" + end check + store first_record as source_records[0] + store model_spec as first_record.model_spec + store valid_model as orm_require_model of model_spec + store relation_spec as orm_relationship_named of model_spec and relation_name + store valid_related as orm_require_model of relation_spec.related_model + store unique_keys as [] + for each source_record in source_records: + check if source_record.model_spec.table_name is not equal to model_spec.table_name: + call orm_fail with "relationship" and "a load batch must contain records from one model" + end check + store key_value as orm_get of source_record and relation_spec.local_field + check if (isnothing of key_value) is equal to no: + check if (contains of unique_keys and key_value) is equal to no: + push with unique_keys and key_value + end check + end check + end for + store collected as [] + store key_offset as 0 + repeat while key_offset is less than (length of unique_keys): + store batch_keys as [] + repeat while key_offset is less than (length of unique_keys) and (length of batch_keys) is less than 100: + push with batch_keys and unique_keys[key_offset] + change key_offset to key_offset plus 1 + end repeat + store requested as orm_query of relation_spec.related_model + call orm_where with requested and (orm_compare of relation_spec.related_field and "in" and batch_keys) + store page_offset as 0 + store has_more as yes + repeat while has_more: + call orm_page with requested and 1000 and page_offset + store related_page as orm_find of session and requested + for each related_record in related_page: + check if (length of collected) is greater than or equal to 10000: + call orm_fail with "relationship" and "eager loading is limited to 10000 related records; query the related model in explicit pages" + end check + push with collected and related_record + end for + change has_more to (length of related_page) is equal to 1000 + change page_offset to page_offset plus 1000 + end repeat + end repeat + // Assemble every result first. A validation/query error cannot leave only + // part of the source batch marked as loaded. + store pending_relations as [] + for each source_record in source_records: + store local_value as orm_get of source_record and relation_spec.local_field + store related_records as [] + check if (isnothing of local_value) is equal to no: + for each related_record in collected: + store related_value as orm_get of related_record and relation_spec.related_field + check if related_value is equal to local_value: + push with related_records and related_record + end check + end for + end check + check if relation_spec.many is equal to no and (length of related_records) is greater than 1: + call orm_fail with "relationship" and "a single-record relationship matched multiple rows; restore the declared unique constraint" + end check + create new OrmLoadedRelation as loaded: + relation_name is relation_name + related_records is related_records + many is relation_spec.many + end + push with pending_relations and loaded + end for + store source_index as 0 + for each source_record in source_records: + store revised_relations as [] + for each existing_relation in source_record.loaded_relations: + check if existing_relation.relation_name is not equal to relation_name: + push with revised_relations and existing_relation + end check + end for + push with revised_relations and pending_relations[source_index] + source_record.replace_relations(revised_relations) + change source_index to source_index plus 1 + end for + return source_records +end action diff --git a/lib/orm/schema.wfl b/lib/orm/schema.wfl new file mode 100644 index 0000000..ae21f30 --- /dev/null +++ b/lib/orm/schema.wfl @@ -0,0 +1,329 @@ +include from "relationships.wfl" + +// Physical schema is separate from logical record relationships. Historical +// migrations own these declarations; current application models do not edit them. +create container OrmForeignKey: + property field_names: List defaults [] + property target_table: Text defaults "" + property target_fields: List defaults [] + property on_delete: Text defaults "NO ACTION" + property on_update: Text defaults "NO ACTION" +end + +create container OrmSchemaTable: + property model_spec + property foreign_keys: List defaults [] +end + +define action called orm_schema_table with parameters model_spec: + store checked_model as orm_require_model of model_spec + create new OrmSchemaTable as table_spec: + model_spec is model_spec + end + return table_spec +end action + +define action called orm_schema_literal with parameters literal_value: + check if literal_value is nothing: + return "NULL" + end check + check if (typeof of literal_value) is equal to "Text": + return "'" with (replace "'" with "''" in literal_value) with "'" + end check + check if (typeof of literal_value) is equal to "Boolean": + check if literal_value: + return "1" + end check + return "0" + end check + check if (typeof of literal_value) is equal to "Number": + return "" with literal_value + end check + call orm_fail with "schema" and "defaults must be text, a finite number, boolean, or NULL" +end action + +define action called orm_schema_column with parameters field_spec: + store field_problem as orm_field_problem of field_spec + check if field_problem is not equal to "": + call orm_fail with "schema field" and field_problem + end check + store type_text as "TEXT" + check if field_spec.value_type is equal to "integer" or field_spec.value_type is equal to "identity" or field_spec.value_type is equal to "boolean": + change type_text to "INTEGER" + end check + check if field_spec.value_type is equal to "number": + change type_text to "REAL" + end check + store column_sql as (orm_quote of field_spec.field_name) with " " with type_text + check if field_spec.primary_key: + change column_sql to column_sql with " PRIMARY KEY" + end check + check if field_spec.generated: + change column_sql to column_sql with " AUTOINCREMENT" + end check + check if field_spec.unique_value: + change column_sql to column_sql with " UNIQUE" + end check + // SQLite's generated INTEGER PRIMARY KEY already rejects NULL storage. + // Omitting redundant NOT NULL preserves the original application schema. + check if field_spec.nullable is equal to no and field_spec.generated is equal to no: + change column_sql to column_sql with " NOT NULL" + end check + check if field_spec.has_default: + change column_sql to column_sql with " DEFAULT " with (orm_schema_literal of field_spec.default_value) + end check + check if field_spec.server_default is equal to "current timestamp": + change column_sql to column_sql with " DEFAULT (datetime('now'))" + end check + return column_sql +end action + +define action called orm_schema_names with parameters field_names: + store quoted_names as [] + for each field_name in field_names: + push with quoted_names and (orm_quote of field_name) + end for + return join of quoted_names and ", " +end action + +define action called orm_schema_create_sql with parameters table_spec and actual_name: + store model_spec as table_spec.model_spec + store valid_model as orm_require_model of model_spec + store clauses as [] + for each field_spec in model_spec.fields: + push with clauses and (orm_schema_column of field_spec) + end for + for each foreign_spec in table_spec.foreign_keys: + check if (length of foreign_spec.field_names) is equal to 0 or (length of foreign_spec.field_names) is not equal to (length of foreign_spec.target_fields): + call orm_fail with "schema foreign key" and "declare equal nonempty local and target field lists" + end check + for each local_name in foreign_spec.field_names: + check if (orm_field_named of model_spec and local_name) is nothing: + call orm_fail with "schema foreign key" and "declare each local field on the table" + end check + end for + store allowed_actions as ["NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT"] + check if (contains of allowed_actions and foreign_spec.on_delete) is equal to no or (contains of allowed_actions and foreign_spec.on_update) is equal to no: + call orm_fail with "schema foreign key" and "use a supported SQLite referential action" + end check + push with clauses and ("FOREIGN KEY (" with (orm_schema_names of foreign_spec.field_names) with ") REFERENCES " with (orm_quote of foreign_spec.target_table) with " (" with (orm_schema_names of foreign_spec.target_fields) with ") ON DELETE " with foreign_spec.on_delete with " ON UPDATE " with foreign_spec.on_update) + end for + return "CREATE TABLE " with (orm_quote of actual_name) with " (" with (join of clauses and ", ") with ")" +end action + +define action called orm_schema_index_sql with parameters table_spec and index_spec: + store unique_text as "" + check if index_spec.unique_values: + change unique_text to "UNIQUE " + end check + return "CREATE " with unique_text with "INDEX " with (orm_quote of index_spec.index_name) with " ON " with (orm_quote of table_spec.model_spec.table_name) with " (" with (orm_schema_names of index_spec.field_names) with ")" +end action + +// Compare authored schema conservatively. Whitespace, identifier quoting and +// ASCII case outside string literals are immaterial; literal bytes are not. +// Extra constraints, column clauses, collation, STRICT and WITHOUT ROWID remain +// visible. Unknown declarations are rejected, never silently normalized away. +define action called orm_schema_normalize with parameters sql_text: + store tokens as [] + store word_text as "" + store source_index as 0 + store source_length as length of sql_text + repeat while source_index is less than source_length: + store next_char as substring of sql_text and source_index and 1 + change source_index to source_index plus 1 + check if contains of "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$" and next_char: + change word_text to word_text with (tolowercase of next_char) + otherwise: + check if word_text is not equal to "": + push with tokens and word_text + change word_text to "" + end check + check if contains of "'\"`[" and next_char: + store quote_end as next_char + check if next_char is equal to "[": + change quote_end to "]" + end check + store quoted_text as "" + store closed_quote as no + repeat while source_index is less than source_length: + store quoted_char as substring of sql_text and source_index and 1 + change source_index to source_index plus 1 + check if quoted_char is equal to quote_end: + check if source_index is less than source_length and (substring of sql_text and source_index and 1) is equal to quote_end and quote_end is not equal to "]": + change quoted_text to quoted_text with quoted_char + change source_index to source_index plus 1 + otherwise: + change closed_quote to yes + break + end check + otherwise: + change quoted_text to quoted_text with quoted_char + end check + end repeat + check if closed_quote is equal to no: + call orm_fail with "schema inspection" and "unterminated quoted schema token; repair the schema before migration" + end check + check if next_char is equal to "'": + push with tokens and ("literal:" with (stringify_json of quoted_text)) + otherwise: + check if (orm_identifier_problem of quoted_text) is equal to "": + push with tokens and (tolowercase of quoted_text) + otherwise: + push with tokens and ("identifier:" with (stringify_json of quoted_text)) + end check + end check + otherwise: + check if (contains of " \r\n\t;" and next_char) is equal to no: + push with tokens and next_char + end check + end check + end check + end repeat + check if word_text is not equal to "": + push with tokens and word_text + end check + return stringify_json of tokens +end action + +define action called orm_schema_exists with parameters session and table_name: + store rows_found as orm_sql_query of session and "SELECT 1 AS present FROM sqlite_schema WHERE type='table' AND name=? COLLATE NOCASE" and [table_name] + return (length of rows_found) is greater than 0 +end action + +define action called orm_schema_problem with parameters session and table_spec: + store table_name as table_spec.model_spec.table_name + store rows_found as orm_sql_query of session and "SELECT sql FROM sqlite_schema WHERE type='table' AND name=? COLLATE NOCASE" and [table_name] + check if (length of rows_found) is not equal to 1: + return "missing managed table " with table_name + end check + store schema_row as rows_found[0] + store expected_sql as orm_schema_create_sql of table_spec and table_name + check if (orm_schema_normalize of schema_row["sql"]) is not equal to (orm_schema_normalize of expected_sql): + return "managed table " with table_name with " differs from its immutable migration declaration; restore the expected schema or author a reviewed migration" + end check + for each index_spec in table_spec.model_spec.indexes: + store index_rows as orm_sql_query of session and "SELECT sql FROM sqlite_schema WHERE type='index' AND name=? COLLATE NOCASE AND tbl_name=? COLLATE NOCASE" and [index_spec.index_name, table_name] + check if (length of index_rows) is not equal to 1: + return "missing managed index " with index_spec.index_name + end check + store index_row as index_rows[0] + check if (orm_schema_normalize of index_row["sql"]) is not equal to (orm_schema_normalize of (orm_schema_index_sql of table_spec and index_spec)): + return "managed index " with index_spec.index_name with " differs from its immutable migration declaration" + end check + end for + return "" +end action + +define action called orm_schema_require with parameters session and table_specs: + for each table_spec in table_specs: + store schema_problem as orm_schema_problem of session and table_spec + check if schema_problem is not equal to "": + call orm_fail with "schema drift" and schema_problem + end check + end for + return yes +end action + +define action called orm_schema_create with parameters session and table_spec: + store created_table as orm_sql_execute of session and (orm_schema_create_sql of table_spec and table_spec.model_spec.table_name) and [] + for each index_spec in table_spec.model_spec.indexes: + store created_index as orm_sql_execute of session and (orm_schema_index_sql of table_spec and index_spec) and [] + end for + return yes +end action + +// Call only within the engine's native schema transaction. Explicit source and +// target columns preserve NULL/empty text and exact SQL integers without a WFL +// round-trip. Extension indexes/triggers are recreated from SQLite's own SQL. +define action called orm_schema_rebuild with parameters session and before_table and after_table and source_fields and target_fields: + store table_name as before_table.model_spec.table_name + check if table_name is not equal to after_table.model_spec.table_name: + call orm_fail with "schema rebuild" and "rebuild keeps the table name; author a separate explicit rename migration" + end check + check if (length of source_fields) is equal to 0 or (length of source_fields) is not equal to (length of target_fields): + call orm_fail with "schema rebuild" and "declare matching nonempty source and target copy columns" + end check + for each field_name in source_fields: + check if (orm_field_named of before_table.model_spec and field_name) is nothing: + call orm_fail with "schema rebuild" and "copy source field is not declared in the prior schema" + end check + end for + for each field_name in target_fields: + check if (orm_field_named of after_table.model_spec and field_name) is nothing: + call orm_fail with "schema rebuild" and "copy target field is not declared in the next schema" + end check + end for + store unique_sources as [] + store unique_targets as [] + store mapping_position as 0 + for each field_name in source_fields: + store target_name as target_fields[mapping_position] + check if (contains of unique_sources and field_name) or (contains of unique_targets and target_name): + call orm_fail with "schema rebuild" and "copy mappings must not repeat source or target fields" + end check + push with unique_sources and field_name + push with unique_targets and target_name + check if (orm_field_named of after_table.model_spec and field_name) is not nothing and target_name is not equal to field_name: + call orm_fail with "schema rebuild" and "retained fields must copy to their same named destination; declare renames by removing the old field from the next schema" + end check + change mapping_position to mapping_position plus 1 + end for + for each prior_field in before_table.model_spec.fields: + check if (orm_field_named of after_table.model_spec and prior_field.field_name) is not nothing and (contains of source_fields and prior_field.field_name) is equal to no: + call orm_fail with "schema rebuild" and "every retained field needs an explicit copy mapping; omission would discard existing values" + end check + end for + store prior_problem as orm_schema_problem of session and before_table + check if prior_problem is not equal to "": + call orm_fail with "schema rebuild" and prior_problem + end check + store extension_objects as orm_sql_query of session and "SELECT type,name,sql FROM sqlite_schema WHERE tbl_name=? COLLATE NOCASE AND type IN ('index','trigger') AND sql IS NOT NULL ORDER BY type,name" and [table_name] + // SQLite permits creating a trigger that later references an absent column. + // Without a SQL dependency parser, refuse that ambiguous case before DROP. + for each prior_field in before_table.model_spec.fields: + check if (orm_field_named of after_table.model_spec and prior_field.field_name) is nothing: + for each extension_object in extension_objects: + check if extension_object["type"] is equal to "trigger": + call orm_fail with "schema rebuild" and "removing columns from a table with extension triggers requires an explicit reviewed trigger migration before the rebuild" + end check + end for + end check + end for + store temporary_name as "_orm_rebuild_" with table_name + check if orm_schema_exists of session and temporary_name: + call orm_fail with "schema rebuild" and "the reserved rebuild table already exists; investigate interrupted or unrelated manual changes" + end check + store old_sequence as nothing + check if orm_schema_exists of session and "sqlite_sequence": + store sequence_rows as orm_sql_query of session and "SELECT CAST(seq AS TEXT) AS seq FROM sqlite_sequence WHERE name=? COLLATE NOCASE" and [table_name] + check if (length of sequence_rows) is greater than 0: + store sequence_row as sequence_rows[0] + change old_sequence to sequence_row["seq"] + end check + end check + store created_table as orm_sql_execute of session and (orm_schema_create_sql of after_table and temporary_name) and [] + store copied_rows as orm_sql_execute of session and ("INSERT INTO " with (orm_quote of temporary_name) with " (" with (orm_schema_names of target_fields) with ") SELECT " with (orm_schema_names of source_fields) with " FROM " with (orm_quote of table_name)) and [] + store dropped_table as orm_sql_execute of session and ("DROP TABLE " with (orm_quote of table_name)) and [] + store renamed_table as orm_sql_execute of session and ("ALTER TABLE " with (orm_quote of temporary_name) with " RENAME TO " with (orm_quote of table_name)) and [] + for each index_spec in after_table.model_spec.indexes: + store created_index as orm_sql_execute of session and (orm_schema_index_sql of after_table and index_spec) and [] + end for + for each extension_object in extension_objects: + store managed_index as no + for each index_spec in before_table.model_spec.indexes: + check if (tolowercase of extension_object["name"]) is equal to (tolowercase of index_spec.index_name): + change managed_index to yes + end check + end for + check if managed_index is equal to no: + store recreated_object as orm_sql_execute of session and extension_object["sql"] and [] + end check + end for + check if old_sequence is not nothing: + store next_key as orm_primary_field of after_table.model_spec + check if next_key.generated: + store kept_sequence as orm_sql_execute of session and "UPDATE sqlite_sequence SET seq=MAX(seq,CAST(? AS INTEGER)) WHERE name=?" and [old_sequence, table_name] + end check + end check + return yes +end action diff --git a/lib/orm/types.wfl b/lib/orm/types.wfl new file mode 100644 index 0000000..e658874 --- /dev/null +++ b/lib/orm/types.wfl @@ -0,0 +1,191 @@ +// Shared declarations and non-mutating validation. A problem is empty text on +// success, otherwise safe corrective guidance; it never contains a field value. +// Public record/database operations raise these problems rather than returning +// a success-shaped result. This module has no application dependencies. + +create container OrmField: + property field_name: Text defaults "" + property value_type: Text defaults "text" + property nullable: Boolean defaults no + property primary_key: Boolean defaults no + property generated: Boolean defaults no + property unique_value: Boolean defaults no + property sensitive: Boolean defaults no + property has_default: Boolean defaults no + // The value is polymorphic; orm_value_problem enforces the declared type. + property default_value defaults nothing + // Server defaults are limited to a named, audited expression vocabulary. + property server_default: Text defaults "" +end + +create container OrmIndex: + property index_name: Text defaults "" + property field_names: List defaults [] + property unique_values: Boolean defaults no +end + +create container OrmRelationship: + property relation_name: Text defaults "" + property related_model defaults nothing + property local_field: Text defaults "" + property related_field: Text defaults "" + property many: Boolean defaults no +end + +define action called orm_identifier_problem with parameters identifier: + store guidance as "use an ASCII letter or underscore followed by ASCII letters, digits, or underscores" + check if (typeof of identifier) is not equal to "Text": + return guidance + end check + check if (length of identifier) is equal to 0: + return guidance + end check + store initials as "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_" + store remaining as initials with "0123456789" + store initial as substring of identifier and 0 and 1 + check if (contains of initials and initial) is equal to no: + return guidance + end check + count from 0 to (length of identifier) minus 1: + store name_char as substring of identifier and count and 1 + check if (contains of remaining and name_char) is equal to no: + return guidance + end check + end count + return "" +end action + +// Compare decimal magnitudes without passing an identity through floating point. +define action called orm_decimal_exceeds with parameters magnitude and upper_bound: + check if (length of magnitude) is greater than (length of upper_bound): + return yes + end check + check if (length of magnitude) is less than (length of upper_bound): + return no + end check + store digits as "0123456789" + count from 0 to (length of magnitude) minus 1: + store digit_a as substring of magnitude and count and 1 + store digit_b as substring of upper_bound and count and 1 + store value_a as index_of of digits and digit_a + store value_b as index_of of digits and digit_b + check if value_a is greater than value_b: + return yes + end check + check if value_a is less than value_b: + return no + end check + end count + return no +end action + +define action called orm_identity_problem with parameters identity_text: + store guidance as "use canonical integer text without spaces, a plus sign, or leading zeros" + check if (typeof of identity_text) is not equal to "Text": + return guidance + end check + store magnitude as identity_text + store upper_bound as "9223372036854775807" + check if (substring of magnitude and 0 and 1) is equal to "-": + change magnitude to substring of magnitude and 1 and ((length of magnitude) minus 1) + change upper_bound to "9223372036854775808" + check if magnitude is equal to "0": + return guidance + end check + end check + check if (length of magnitude) is equal to 0: + return guidance + end check + check if (length of magnitude) is greater than 1 and (substring of magnitude and 0 and 1) is equal to "0": + return guidance + end check + count from 0 to (length of magnitude) minus 1: + store digit_char as substring of magnitude and count and 1 + check if (contains of "0123456789" and digit_char) is equal to no: + return guidance + end check + end count + check if orm_decimal_exceeds of magnitude and upper_bound: + return "identity is outside SQLite's signed 64-bit range" + end check + return "" +end action + +define action called orm_value_problem with parameters field_spec and field_value: + check if isnothing of field_value: + check if field_spec.nullable: + return "" + end check + return "null is not allowed; supply a value or omit the field to use its declared default" + end check + store actual_type as typeof of field_value + check if field_spec.value_type is equal to "text": + check if actual_type is equal to "Text": + return "" + end check + return "expected text" + end check + check if field_spec.value_type is equal to "identity": + return orm_identity_problem of field_value + end check + check if field_spec.value_type is equal to "integer": + check if actual_type is equal to "Number": + check if field_value is greater than or equal to -9007199254740991 and field_value is less than or equal to 9007199254740991: + check if (floor of field_value) is equal to field_value: + return "" + end check + end check + end check + return "expected a whole number in the exactly representable range" + end check + check if field_spec.value_type is equal to "number": + check if actual_type is equal to "Number": + // NaN differs from itself; infinity minus itself is NaN. + check if (field_value minus field_value) is equal to 0: + return "" + end check + end check + return "expected a finite number" + end check + check if field_spec.value_type is equal to "boolean": + check if actual_type is equal to "Boolean": + return "" + end check + return "expected boolean" + end check + return "supported field types are text, integer, number, boolean, and identity" +end action + +define action called orm_field_problem with parameters field_spec: + store name_problem as orm_identifier_problem of field_spec.field_name + check if name_problem is not equal to "": + return name_problem + end check + store supported_types as ["text" and "integer" and "number" and "boolean" and "identity"] + check if (contains of supported_types and field_spec.value_type) is equal to no: + return "supported field types are text, integer, number, boolean, and identity" + end check + check if field_spec.generated: + check if field_spec.value_type is not equal to "identity" or field_spec.primary_key is equal to no or field_spec.nullable: + return "generated fields must be non-null identity primary keys" + end check + check if field_spec.has_default or field_spec.server_default is not equal to "": + return "generated identities cannot declare a default" + end check + end check + check if field_spec.has_default: + check if field_spec.server_default is not equal to "": + return "choose a literal default or a server default, not both" + end check + store default_problem as orm_value_problem of field_spec and field_spec.default_value + check if default_problem is not equal to "": + return "invalid default: " with default_problem + end check + end check + check if field_spec.server_default is not equal to "": + check if field_spec.server_default is not equal to "current timestamp" or field_spec.value_type is not equal to "text": + return "the supported server default is current timestamp on a text field" + end check + end check + return "" +end action diff --git a/lib/orm/writes.wfl b/lib/orm/writes.wfl new file mode 100644 index 0000000..64db909 --- /dev/null +++ b/lib/orm/writes.wfl @@ -0,0 +1,176 @@ +include from "connections.wfl" + +define action called orm_insert_mode with parameters session and record_value and conflict_mode and conflict_field: + store record_problem as orm_record_problem of record_value and yes + check if record_problem is not equal to "": + call orm_fail with "insert" and record_problem + end check + store model_spec as record_value.model_spec + store field_parts as [] + store placeholders as [] + store bound_values as [] + store updated_parts as [] + for each assignment in record_value.assigned_values: + store field_spec as orm_require_field of model_spec and assignment.field_name + store quoted_name as orm_quote of field_spec.field_name + push with field_parts and quoted_name + push with placeholders and "?" + push with bound_values and assignment.field_value + check if field_spec.primary_key is equal to no and field_spec.field_name is not equal to conflict_field: + push with updated_parts and (quoted_name with " = excluded." with quoted_name) + end check + end for + store sql_text as "INSERT INTO " with (orm_quote of model_spec.table_name) + check if (length of field_parts) is equal to 0: + change sql_text to sql_text with " DEFAULT VALUES" + otherwise: + change sql_text to sql_text with " (" with (join of field_parts and ", ") with ") VALUES (" with (join of placeholders and ", ") with ")" + end check + check if conflict_mode is not equal to "error": + store conflict_spec as orm_require_field of model_spec and conflict_field + check if conflict_spec.primary_key is equal to no and conflict_spec.unique_value is equal to no: + call orm_fail with "insert conflict" and "the conflict field must be a primary or unique field" + end check + check if (orm_has of record_value and conflict_field) is equal to no: + call orm_fail with "insert conflict" and "assign the conflict field explicitly" + end check + change sql_text to sql_text with " ON CONFLICT (" with (orm_quote of conflict_field) with ")" + check if conflict_mode is equal to "ignore": + change sql_text to sql_text with " DO NOTHING" + otherwise: + check if conflict_mode is not equal to "update" or (length of updated_parts) is equal to 0: + call orm_fail with "insert conflict" and "an upsert requires at least one assigned non-key field to update" + end check + change sql_text to sql_text with " DO UPDATE SET " with (join of updated_parts and ", ") + end check + end check + store rowid_alias as "_orm_insert_rowid" + repeat while (orm_field_named of model_spec and rowid_alias) is not nothing: + change rowid_alias to rowid_alias with "_" + end repeat + change sql_text to sql_text with " RETURNING " with (orm_projection_sql of model_spec and []) with ", CAST(" with (orm_quote of (orm_rowid_name of model_spec)) with " AS TEXT) AS " with (orm_quote of rowid_alias) + store native_rows as orm_read_sql of session and (orm_sql of sql_text and bound_values and "unknown") + check if (length of native_rows) is equal to 0: + return nothing + end check + store inserted_row as native_rows[0] + store saved as orm_map_row of model_spec and inserted_row + saved.remember_insert(inserted_row[rowid_alias]) + return saved +end action + +define action called orm_insert with parameters session and record_value: + return orm_insert_mode of session and record_value and "error" and "" +end action + +define action called orm_insert_if_absent with parameters session and record_value and conflict_field: + return orm_insert_mode of session and record_value and "ignore" and conflict_field +end action + +define action called orm_upsert with parameters session and record_value and conflict_field: + return orm_insert_mode of session and record_value and "update" and conflict_field +end action + +define action called orm_save with parameters session and record_value: + check if record_value.persisted is equal to no: + return orm_insert of session and record_value + end check + store record_problem as orm_record_problem of record_value and no + check if record_problem is not equal to "": + call orm_fail with "save" and record_problem + end check + store model_spec as record_value.model_spec + store key_spec as orm_primary_field of model_spec + check if (isnothing of record_value.original_key) or (orm_has of record_value and key_spec.field_name) is equal to no: + call orm_fail with "save" and "a persisted record needs a non-null projected primary key; reload the complete record" + end check + check if (orm_get of record_value and key_spec.field_name) is not equal to record_value.original_key: + call orm_fail with "save" and "changing a persisted primary key is not supported; use an explicit data migration" + end check + store assignments_sql as [] + store bound_values as [] + for each assignment in record_value.assigned_values: + check if assignment.field_name is not equal to key_spec.field_name: + push with assignments_sql and ((orm_quote of assignment.field_name) with " = ?") + push with bound_values and assignment.field_value + end check + end for + check if (length of assignments_sql) is equal to 0: + call orm_fail with "save" and "there are no assigned non-key fields; assign a field or reload the complete record" + end check + push with bound_values and record_value.original_key + store sql_text as "UPDATE " with (orm_quote of model_spec.table_name) with " SET " with (join of assignments_sql and ", ") with " WHERE " with (orm_quote of key_spec.field_name) with " = ? RETURNING " with (orm_projection_sql of model_spec and []) + store native_rows as orm_read_sql of session and (orm_sql of sql_text and bound_values and "unknown") + check if (length of native_rows) is equal to 0: + call orm_fail with "save" and "the persisted record no longer exists; reload it before deciding whether to create a replacement" + end check + return orm_map_row of model_spec and native_rows[0] +end action + +// Bulk mutation honors the same page/order as a read query. A predicate or an +// explicit whole-table intent is still required even though each page is bounded. +define action called orm_write_selection with parameters requested: + store filter_sql as orm_compile_predicate of requested.model_spec and requested.predicate + check if filter_sql.constant_kind is equal to "true" and requested.whole_table is equal to no: + call orm_fail with "bulk write" and "an unrestricted write needs explicit orm_whole_table intent" + end check + store key_spec as orm_primary_field of requested.model_spec + create new OrmQuery as selection: + model_spec is requested.model_spec + predicate is requested.predicate + ordering is requested.ordering + selected_fields is [key_spec.field_name] + page_limit is requested.page_limit + page_offset is requested.page_offset + end + store selection_kind as "records" + store selected_name as key_spec.field_name + check if key_spec.nullable: + change selection_kind to "row identity" + change selected_name to orm_rowid_name of requested.model_spec + end check + store selected_sql as orm_compile_select of selection and selection_kind + return orm_sql of ((orm_quote of selected_name) with " IN (" with selected_sql.sql_text with ")") and selected_sql.bound_values and filter_sql.constant_kind +end action + +define action called orm_update with parameters session and requested and assignments: + store draft as orm_record of requested.model_spec + call orm_assign with draft and assignments + return orm_update_record of session and requested and draft +end action + +// Trusted single-field counterpart to orm_set_sensitive. Callers authorize the +// specific field/value; unrestricted query and primary-key guards still apply. +define action called orm_update_sensitive with parameters session and requested and field_name and field_value: + store draft as orm_record of requested.model_spec + call orm_set_sensitive with draft and field_name and field_value + return orm_update_record of session and requested and draft +end action + +define action called orm_update_record with parameters session and requested and draft: + store key_spec as orm_primary_field of requested.model_spec + store assignment_parts as [] + store bound_values as [] + for each assignment in draft.assigned_values: + check if assignment.field_name is equal to key_spec.field_name: + call orm_fail with "bulk update" and "changing primary keys requires an explicit data migration" + end check + push with assignment_parts and ((orm_quote of assignment.field_name) with " = ?") + push with bound_values and assignment.field_value + end for + check if (length of assignment_parts) is equal to 0: + call orm_fail with "bulk update" and "assign at least one field" + end check + store selection_sql as orm_write_selection of requested + for each bound_value in selection_sql.bound_values: + push with bound_values and bound_value + end for + store sql_text as "UPDATE " with (orm_quote of requested.model_spec.table_name) with " SET " with (join of assignment_parts and ", ") with " WHERE " with selection_sql.sql_text + return orm_write_sql of session and (orm_sql of sql_text and bound_values and selection_sql.constant_kind) +end action + +define action called orm_delete with parameters session and requested: + store selection_sql as orm_write_selection of requested + store sql_text as "DELETE FROM " with (orm_quote of requested.model_spec.table_name) with " WHERE " with selection_sql.sql_text + return orm_write_sql of session and (orm_sql of sql_text and selection_sql.bound_values and selection_sql.constant_kind) +end action diff --git a/main.wfl b/main.wfl index 90d392e..4b4ea67 100644 --- a/main.wfl +++ b/main.wfl @@ -538,7 +538,7 @@ define action called handle_post_create with parameters db and req and req_body end action define action called handle_post_edit with parameters db and req and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store post_row as post_by_id of db and the_id check if post_row is nothing: call handle_404 with db and req and user @@ -554,7 +554,7 @@ define action called handle_post_edit with parameters db and req and user and id end action define action called handle_post_update with parameters db and req and req_body and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store post_row as post_by_id of db and the_id check if post_row is nothing: call handle_404 with db and req and user @@ -580,7 +580,7 @@ define action called handle_post_update with parameters db and req and req_body end action define action called handle_post_delete with parameters db and req and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store post_row as post_by_id of db and the_id check if post_row is nothing: call handle_404 with db and req and user @@ -680,7 +680,7 @@ define action called handle_page_create with parameters db and req and req_body end action define action called handle_page_edit with parameters db and req and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store page_row as page_by_id of db and the_id check if page_row is nothing: call handle_404 with db and req and user @@ -696,7 +696,7 @@ define action called handle_page_edit with parameters db and req and user and id end action define action called handle_page_update with parameters db and req and req_body and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store page_row as page_by_id of db and the_id check if page_row is nothing: call handle_404 with db and req and user @@ -722,7 +722,7 @@ define action called handle_page_update with parameters db and req and req_body end action define action called handle_page_delete with parameters db and req and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store page_row as page_by_id of db and the_id check if page_row is nothing: call handle_404 with db and req and user @@ -826,7 +826,7 @@ define action called handle_user_create with parameters db and req and req_body end action define action called handle_user_edit with parameters db and req and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store edit_user as user_by_id of db and the_id check if edit_user is nothing: call send_text_status with req and "User not found" and 404 @@ -839,7 +839,7 @@ define action called handle_user_edit with parameters db and req and user and id end action define action called handle_user_update with parameters db and req and req_body and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store edit_user as user_by_id of db and the_id check if edit_user is nothing: call send_text_status with req and "User not found" and 404 @@ -861,7 +861,7 @@ define action called handle_user_update with parameters db and req and req_body end action define action called handle_user_delete with parameters db and req and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text check if user["id"] is equal to the_id: call send_text_status with req and "You cannot delete your own account." and 400 return yes @@ -1035,7 +1035,7 @@ define action called handle_media_upload with parameters db and req and user: end action define action called handle_media_delete with parameters db and req and user and id_text: - store the_id as to_int of id_text and 0 + store the_id as db_route_identity of id_text store media_row as media_by_id of db and the_id check if media_row is nothing: call handle_404 with db and req and user diff --git a/scripts/.wflcfg b/scripts/.wflcfg new file mode 100644 index 0000000..f6245b0 --- /dev/null +++ b/scripts/.wflcfg @@ -0,0 +1,9 @@ +# Default configured budget; the complete command explicitly supplies +# --execution-timeout 1200 before the script path. Child waits remain bounded. +timeout_seconds = 300 +execution_logging = false +debug_report_enabled = false +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true +max_buffer_size_bytes = 8388608 diff --git a/scripts/migrate.wfl b/scripts/migrate.wfl new file mode 100644 index 0000000..1e6a1e7 --- /dev/null +++ b/scripts/migrate.wfl @@ -0,0 +1,220 @@ +// Run from the application root, exactly like main.wfl. Configuration is read +// using the same parser; read-only commands never create a database/directory. +include from "../app/migrations.wfl" + +define action called migration_cli_usage: + display "Usage: wfl scripts/migrate.wfl status|plan|up [--target ID|latest] [--database PATH]" + display " wfl scripts/migrate.wfl down --count N|--target ID|zero [--database PATH]" + display " wfl scripts/migrate.wfl new ascii_name" + return yes +end action + +define action called migration_cli_show with parameters migration_plan: + display "Applied migrations: " with (length of migration_plan.applied) + for each applied_row in migration_plan.applied: + display " applied " with applied_row["migration_id"] with " " with applied_row["migration_name"] + end for + display "Pending migrations: " with (length of migration_plan.pending) + for each pending_id in migration_plan.pending: + display " pending " with pending_id + end for + return yes +end action + +define action called migration_cli_new with parameters migration_name: + check if (orm_identifier_problem of migration_name) is not equal to "": + call orm_fail with "migration scaffold" and "use an ASCII name such as add_post_summary; paths and punctuation are not accepted" + end check + store registry as scriptorium_migration_registry + store known_ids as orm_migration_registry_check of registry + store latest_version as registry[(length of registry) minus 1] + store utc_stamp as utc_now + store version_id as format_datetime of utc_stamp and "%Y%m%d%H%M%S" + check if (orm_decimal_exceeds of version_id and latest_version.migration_id) is equal to no: + call orm_fail with "migration scaffold" and "the clock does not follow the latest version ID; correct the clock before scaffolding" + end check + store migration_dir as path_join of current_directory and "app/migrations" + store previous_file as "" + for each entry_name in (list_dir of migration_dir): + check if entry_name starts with (latest_version.migration_id with "_") and entry_name ends with ".wfl": + check if previous_file is not equal to "": + call orm_fail with "migration scaffold" and "multiple files use the latest version ID; restore an unambiguous registry" + end check + change previous_file to entry_name + end check + check if entry_name starts with (version_id with "_"): + call orm_fail with "migration scaffold" and "a file already uses this second's ID; retry after the clock advances" + end check + end for + check if previous_file is equal to "": + call orm_fail with "migration scaffold" and "the latest registered source file is missing" + end check + store new_name as version_id with "_" with migration_name with ".wfl" + store new_path as path_join of migration_dir and new_name + store source_text as "// New immutable historical migration. Complete and review before registration.\ninclude from \"" with previous_file with "\"\n\ndefine action called migration_" with version_id with " with parameters immutable_source:\n store previous_version as migration_" with latest_version.migration_id with " of (scriptorium_migration_source of \"" with previous_file with "\")\n create new OrmMigration as migration_spec:\n migration_id is \"" with version_id with "\"\n migration_name is \"" with migration_name with "\"\n // Replace these lists with explicit typed up/down steps.\n up_steps is []\n down_steps is []\n irreversible_reason is \"TODO: describe why rollback cannot restore the data, or supply lossless down steps\"\n managed_before is previous_version.managed_after\n // Declare the complete next historical schema, independent of app/models.\n managed_after is previous_version.managed_after\n source_text is immutable_source\n end\n return migration_spec\nend action\n" + open file at new_path for writing as scaffold_file + try: + wait for write content source_text into scaffold_file + finally: + close file scaffold_file + end try + display "Created " with new_path + display "Complete the historical before/after declarations and up/down steps; empty scaffolds are rejected." + display "In app/migrations.wfl replace its final historical include with migrations/" with new_name + display "Append migration_" with version_id with " of (scriptorium_migration_source of \"" with new_name with "\") to scriptorium_migration_registry." + display "Run plan, review the generated SQL in the declaration, test up/down on a copied database, then commit the immutable file and registry." + return yes +end action + +store operation_name as "" +store requested_target as "latest" +store target_given as no +store requested_count as nothing +store database_override as "" +store scaffold_name as "" +store argument_index as 0 +try: + repeat while argument_index is less than (length of args): + store argument_text as args[argument_index] + change argument_index to argument_index plus 1 + check if argument_text is equal to "--help": + call migration_cli_usage + exit program + end check + check if argument_text is equal to "--database" or argument_text is equal to "--target" or argument_text is equal to "--count": + check if argument_index is greater than or equal to (length of args): + call orm_fail with "migration arguments" and ("missing value for " with argument_text) + end check + store argument_value as args[argument_index] + change argument_index to argument_index plus 1 + check if argument_text is equal to "--database": + change database_override to argument_value + end check + check if argument_text is equal to "--target": + change requested_target to argument_value + change target_given to yes + end check + check if argument_text is equal to "--count": + // Decimal validation avoids using a parsing exception as control + // flow and never treats malformed input as the default count. + check if (length of argument_value) is equal to 0 or (length of argument_value) is greater than 6: + call orm_fail with "rollback count" and "use a positive whole migration count" + end check + count from 0 to (length of argument_value) minus 1: + check if (contains of "0123456789" and (substring of argument_value and count and 1)) is equal to no: + call orm_fail with "rollback count" and "use a positive whole migration count" + end check + end count + change requested_count to to_int of argument_value and 0 + end check + otherwise: + check if operation_name is equal to "": + change operation_name to argument_text + otherwise: + check if operation_name is equal to "new" and scaffold_name is equal to "": + change scaffold_name to argument_text + otherwise: + call orm_fail with "migration arguments" and "unexpected argument; use --help for the exact syntax" + end check + end check + end check + end repeat + check if operation_name is equal to "new": + check if database_override is not equal to "" or target_given or requested_count is not nothing: + call orm_fail with "migration arguments" and "new accepts only a migration name" + end check + call migration_cli_new with scaffold_name + exit program + end check + check if (contains of ["status", "plan", "up", "down"] and operation_name) is equal to no: + call migration_cli_usage + call orm_fail with "migration arguments" and "select status, plan, up, down, or new" + end check + check if operation_name is not equal to "down" and requested_count is not nothing: + call orm_fail with "migration arguments" and "--count is only valid with down" + end check + check if operation_name is equal to "down": + check if (target_given and requested_count is not nothing) or (target_given is equal to no and requested_count is nothing): + call orm_fail with "migration arguments" and "down requires exactly one of --count or --target" + end check + end check + store cfg_text as "" + check if file exists at ".wflcfg": + open file at ".wflcfg" for reading as cfg_file + try: + wait for change cfg_text to read content from cfg_file + finally: + close file cfg_file + end try + end check + store data_root as config_value_from of cfg_text and "data_dir" and "" + store database_path as path_join of current_directory and "scriptorium.db" + check if data_root is not equal to "": + change database_path to path_join of current_directory and data_root and "scriptorium.db" + end check + check if database_override is not equal to "": + change database_path to path_join of current_directory and database_override + end check + display "Database target: " with database_path + store registry as scriptorium_migration_registry + store checked_registry as orm_migration_registry_check of registry + store target_position as orm_migration_target of registry and requested_target + check if (file exists at database_path) is equal to no: + check if operation_name is equal to "status" or operation_name is equal to "plan": + display "Database absent; no file or directory created." + display "Applied migrations: 0" + display "Planned fresh apply: " with target_position with " migrations" + exit program + end check + check if operation_name is equal to "down": + call orm_fail with "migration target" and "database does not exist; rollback has no applied history" + end check + call makedirs with (path_dirname of database_path) + end check + store session as orm_open of database_path + try: + store history_rows as orm_migration_history of session and registry + store has_history as orm_schema_exists of session and "_orm_migrations" + store legacy_version as 0 + check if has_history is equal to no: + change legacy_version to scriptorium_legacy_version of session and registry + end check + check if operation_name is equal to "status" or operation_name is equal to "plan": + check if legacy_version is greater than 0: + check if target_position is less than legacy_version: + call orm_fail with "migration target" and "target predates the existing legacy schema; adoption cannot downgrade it" + end check + display "Legacy adoption required: version " with legacy_version with "; managed declarations verified. No changes made." + display "Pending after adoption: " with (target_position minus legacy_version) + otherwise: + store planned as orm_migration_plan of session and registry and requested_target + call migration_cli_show with planned + end check + end check + check if operation_name is equal to "up": + check if target_position is less than legacy_version: + call orm_fail with "migration target" and "target predates the existing legacy schema; adoption cannot downgrade it" + end check + store adopted as scriptorium_adopt of session and registry + store applied_plan as orm_migrate of session and registry and requested_target + call migration_cli_show with applied_plan + end check + check if operation_name is equal to "down": + check if has_history is equal to no: + call orm_fail with "rollback" and "legacy schema has no applied history; inspect and explicitly adopt it with up first" + end check + store rolled_plan as nothing + check if requested_count is not nothing: + change rolled_plan to orm_rollback_count of session and registry and requested_count + otherwise: + change rolled_plan to orm_rollback_to of session and registry and requested_target + end check + call migration_cli_show with rolled_plan + end check + finally: + call orm_close with session + end try +when error: + display "Migration failed: " with error_message + exit program with code 1 +end try diff --git a/scripts/resolve_runtime.wfl b/scripts/resolve_runtime.wfl new file mode 100644 index 0000000..bf5b958 --- /dev/null +++ b/scripts/resolve_runtime.wfl @@ -0,0 +1,2 @@ +// Used only to resolve an explicit --wfl program through native PATH launch. +display call current_executable diff --git a/scripts/run_tests.py b/scripts/run_tests.py deleted file mode 100644 index 303810a..0000000 --- a/scripts/run_tests.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -"""Run Scriptorium's WFL test suites.""" - -import argparse -import math -from pathlib import Path -import shutil -import subprocess -import sys -import tempfile - - -def positive_seconds(value): - try: - seconds = float(value) - except ValueError as exc: - raise argparse.ArgumentTypeError("timeout must be a positive number") from exc - if not math.isfinite(seconds) or seconds <= 0: - raise argparse.ArgumentTypeError("timeout must be finite and greater than zero") - return seconds - - -def run_suite(executable, suite, directory, timeout, label): - print(f"\n=== {label} ===", flush=True) - try: - result = subprocess.run( - [executable, "--test", str(suite)], - cwd=directory, - timeout=timeout, - check=False, - ) - except subprocess.TimeoutExpired: - print(f"FAIL {label}: timeout after {timeout:g} seconds", file=sys.stderr, flush=True) - return False - except OSError as exc: - print(f"FAIL {label}: {exc}", file=sys.stderr, flush=True) - return False - if result.returncode != 0: - print(f"FAIL {label}: exit {result.returncode}", file=sys.stderr, flush=True) - return False - print(f"PASS {label}", flush=True) - return True - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--wfl", default="wfl", help="WFL executable (default: wfl on PATH)") - parser.add_argument("--include-scribe", action="store_true", - help="also run the pinned Scribe suite in a temporary copy") - parser.add_argument("--timeout", type=positive_seconds, default=120, - help="maximum seconds per suite (default: 120)") - args = parser.parse_args() - root = Path(__file__).resolve().parent.parent - suites = sorted(path for path in (root / "TestPrograms").rglob("*.test.wfl") - if path.is_file()) - if not suites: - parser.error("no WFL test suites found under TestPrograms/") - executable = shutil.which(args.wfl) - if executable is None: - parser.error(f"WFL executable not found: {args.wfl}") - executable = str(Path(executable).resolve()) - scribe = root / "lib" / "scribe" - if args.include_scribe: - for required in (scribe / "src" / "scribe.wfl", scribe / "tests" / "scribe.test.wfl"): - if not required.is_file(): - parser.error(f"missing Scribe file: {required}; initialize the pinned submodule") - - results = [] - try: - for suite in suites: - label = suite.relative_to(root).as_posix() - results.append(run_suite(executable, suite, root, args.timeout, label)) - if args.include_scribe: - # Upstream tests write build/ fixtures. Never write those into the - # dependency checkout or carry stale output into a test run. - with tempfile.TemporaryDirectory(prefix="scriptorium-scribe-tests-") as temporary: - copy = Path(temporary) / "scribe" - shutil.copytree(scribe, copy, ignore=shutil.ignore_patterns(".git", "build", "__pycache__")) - (copy / "build").mkdir() - results.append(run_suite( - executable, copy / "tests" / "scribe.test.wfl", copy, args.timeout, - "lib/scribe/tests/scribe.test.wfl (temporary copy)", - )) - except OSError as exc: - print(f"Test setup or cleanup failed: {exc}", file=sys.stderr) - return 2 - except KeyboardInterrupt: - print("Test run interrupted", file=sys.stderr) - return 130 - failures = len(results) - sum(results) - print(f"\n{len(results)} suites: {sum(results)} passed, {failures} failed", flush=True) - return 1 if failures else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/run_tests.wfl b/scripts/run_tests.wfl new file mode 100644 index 0000000..fb2db62 --- /dev/null +++ b/scripts/run_tests.wfl @@ -0,0 +1,173 @@ +// Complete suite by default; --group selects an explicitly focused run. +include from "test_support.wfl" + +store repo_root as path_dirname of script_directory +store runtime_path as call current_executable +store suite_timeout as 120 +store selected_group as "all" +store include_scribe as no +store argument_index as 0 + +define action called runner_error with parameters detail: + display "Test setup or cleanup failed: " with detail + exit program with code 2 +end action + +repeat while argument_index is less than (length of args): + store option_name as args[argument_index] + change argument_index to argument_index plus 1 + check if option_name is equal to "--help": + display "Usage: wfl [--execution-timeout SECONDS] scripts/run_tests.wfl [--wfl PROGRAM] [--timeout SECONDS] [--group all|application|tooling|integration|runtime|examples|scribe] [--include-scribe]" + display "Default: all suites, executable examples, and pinned Scribe; 120 seconds per suite." + display "For the complete suite, use --execution-timeout 1200 before the script path." + exit program + otherwise: + check if option_name is equal to "--include-scribe": + change include_scribe to yes + otherwise: + check if argument_index is greater than or equal to (length of args): + call runner_error with ("missing value for " with option_name) + end check + store option_value as args[argument_index] + change argument_index to argument_index plus 1 + check if option_name is equal to "--wfl": + change runtime_path to option_value + otherwise: + check if option_name is equal to "--group": + change selected_group to option_value + otherwise: + check if option_name is equal to "--timeout": + store parsed_timeout as nothing + try: + change parsed_timeout to parse_json of option_value + when error: + call runner_error with "timeout must be a finite positive number" + end try + check if (type_of of parsed_timeout) is not equal to "Number": + call runner_error with "timeout must be a finite positive number" + end check + check if parsed_timeout is less than or equal to 0 or parsed_timeout is greater than 31536000: + call runner_error with "timeout must be positive and at most one year" + end check + change suite_timeout to parsed_timeout + otherwise: + call runner_error with ("unknown option: " with option_name) + end check + end check + end check + end check + end check +end repeat + +store group_paths as parse_json of "{\"application\":\"TestPrograms\",\"tooling\":\"tests/tooling\",\"integration\":\"tests/integration\",\"runtime\":\"tests/runtime\",\"examples\":\"examples\"}" +store group_names as ["application", "tooling", "integration", "runtime", "examples"] +check if selected_group is not equal to "all" and selected_group is not equal to "scribe" and (includes of group_names and selected_group) is equal to no: + call runner_error with ("unknown test group: " with selected_group) +end check +check if selected_group is equal to "all" or selected_group is equal to "scribe": + change include_scribe to yes +end check + +store suites as [] +for each group_name in group_names: + check if selected_group is equal to "all" or selected_group is equal to group_name: + store group_path as path_join of repo_root and group_paths[group_name] + store discovered as [] + check if is_dir of group_path: + change discovered to rglob of "*.test.wfl" and group_path + end check + call sort with discovered + store group_count as 0 + for each suite_path in discovered: + check if is_file of suite_path: + push with suites and suite_path + change group_count to group_count plus 1 + end check + end for + check if group_count is equal to 0: + call runner_error with ("no WFL test suites found under " with group_paths[group_name] with "/") + end check + end check +end for + +store scribe_root as path_join of repo_root and "lib/scribe" +check if include_scribe: + for each required_path in ["src/scribe.wfl", "tests/scribe.test.wfl"]: + check if (is_file of (path_join of scribe_root and required_path)) is equal to no: + call runner_error with ("missing Scribe file: " with required_path with "; initialize the pinned submodule") + end check + end for +end check + +// Validate overrides before starting any suite and pass the resolved identity +// to every suite as args[0], including when invoked outside the repository. +try: + store resolver_path as path_join of script_directory and "resolve_runtime.wfl" + store runtime_outcome as test_execute of runtime_path and [resolver_path] and repo_root and 10 + check if runtime_outcome["success"] is equal to no: + call runner_error with ("WFL executable failed: " with runtime_outcome["error"]) + end check + change runtime_path to trim of runtime_outcome["output"] + check if (is_file of runtime_path) is equal to no: + call runner_error with "WFL executable discovery did not return an existing program" + end check +when error: + call runner_error with ("WFL executable not found or unavailable: " with error_message) +end try + +define action called run_suite with parameters suite_path and working_path and suite_label: + display newline with "=== " with suite_label with " ===" + try: + store outcome as test_execute of runtime_path and ["--test", suite_path, runtime_path] and working_path and suite_timeout + display outcome["output"] + check if (length of outcome["error"]) is greater than 0: + display "Child stderr for " with suite_label with ":" with newline with outcome["error"] + end check + check if outcome["success"] is equal to no: + display "FAIL " with suite_label with ": exit " with outcome["exit_code"] + return no + end check + when error: + display "FAIL " with suite_label with ": " with error_message + return no + end try + display "PASS " with suite_label + return yes +end action + +store passed_suites as 0 +store total_suites as 0 +store disposable_root as path_join of repo_root and "target/test-artifacts/scribe" and (generate_uuid) +try: + for each suite_path in suites: + change total_suites to total_suites plus 1 + check if run_suite of suite_path and repo_root and suite_path: + change passed_suites to passed_suites plus 1 + end check + end for + check if include_scribe: + store copy_root as path_join of disposable_root and "scribe" + call test_copy_tree with scribe_root and copy_root + call makedirs with (path_join of copy_root and "build") + change total_suites to total_suites plus 1 + store scribe_suite as path_join of copy_root and "tests/scribe.test.wfl" + check if run_suite of scribe_suite and copy_root and "lib/scribe/tests/scribe.test.wfl (temporary copy)": + change passed_suites to passed_suites plus 1 + end check + end check +when error: + call runner_error with error_message +finally: + try: + check if is_dir of disposable_root: + call remove_dir with disposable_root and yes + end check + when error: + call runner_error with error_message + end try +end try +store failed_suites as total_suites minus passed_suites +display newline with total_suites with " suites: " with passed_suites with " passed, " with failed_suites with " failed" +check if failed_suites is greater than 0: + exit program with code 1 +end check diff --git a/scripts/test_support.wfl b/scripts/test_support.wfl new file mode 100644 index 0000000..f9f97cd --- /dev/null +++ b/scripts/test_support.wfl @@ -0,0 +1,48 @@ +// Shared WFL test infrastructure. These helpers contain no test discovery. +define action called test_write_text with parameters destination and text_value: + call makedirs with (path_dirname of destination) + open file at destination for writing as output_file + try: + wait for write content text_value into output_file + finally: + close file output_file + end try +end action + +define action called test_read_text with parameters source_path: + open file at source_path for reading as input_file + try: + wait for store text_value as read content from input_file + return text_value + finally: + close file input_file + end try +end action + +define action called test_copy_tree with parameters source_path and destination: + call makedirs with destination + store entries as list_dir of source_path + for each entry_name in entries: + check if entry_name is not equal to ".git" and entry_name is not equal to "build" and entry_name is not equal to "__pycache__": + store child_source as path_join of source_path and entry_name + store child_destination as path_join of destination and entry_name + check if is_dir of child_source: + call test_copy_tree with child_source and child_destination + otherwise: + call copy_file with child_source and child_destination + end check + end check + end for +end action + +// Completion drains both diagnostic streams and releases the owned process. +// The runtime also kills descendants on timeout and on interpreter shutdown. +define action called test_execute with parameters program_path and argument_list and working_path and timeout_seconds: + wait for spawn command program_path with arguments argument_list in directory working_path as child_process + try: + wait for process child_process to complete with timeout timeout_seconds and read result as outcome + return outcome + finally: + close process child_process + end try +end action diff --git a/testing.md b/testing.md index 81139f2..f44245a 100644 --- a/testing.md +++ b/testing.md @@ -5,7 +5,7 @@ governance to this WFL application. It defines both required evidence and the limits of the tooling that exists today. It does not claim that the repository already implements every gate required for a release. -- Policy and profile version: 1.0. +- Policy and profile version: 1.1. - Adopted: 2026-09-12; CI profile updated: 2026-09-20. - Test-suite and infrastructure owner: the Maintainer named in [GOVERNANCE.md](GOVERNANCE.md). @@ -65,9 +65,20 @@ Missing automation does not exempt new or changed behavior from these rules. The application needs WFL with its web server, SQLite, crypto, and testing built-ins, plus the exact Scribe commit recorded at `lib/scribe`. Initialize submodules with `git submodule update --init --recursive`. Python 3.11 or newer -is needed only for repository tooling. Run individual WFL suites from the +and Git are needed to exercise the existing hygiene checker; every test +scenario, fixture, assertion, helper and driver is WFL. Run individual suites from the Scriptorium root so relative includes, templates, and assets resolve correctly. +The ORM and test suites additionally require application errors, pinned schema +transactions, HTTP redirect/header controls and owned process completion. +Official nightly 26.9.16 at `23c1a457` contains those capabilities and the explicit +`--execution-timeout` invocation option required by the complete Linux runner. +Official 26.9.12 lacks the application/test prerequisites; 26.9.14 lacks the +explicit invocation-budget option. +Record source revision as well as the version: +[the verification record](docs/orm-verification.md) identifies the published +artifacts and final CI evidence. + WFL 26.9.3 on Windows is the local adoption baseline: the five application suites passed there on 2026-09-12. This is a recorded observation, not a complete supported-platform matrix or a minimum-version promise. Record `wfl --version`, @@ -75,8 +86,9 @@ OS, Scriptorium revision, and Scribe revision with runtime evidence. No Linux, macOS, alternate WFL version, or production configuration is release-verified merely because these suites passed on Windows. -The five suites use in-memory SQLite and direct action calls; they need no -external credentials or service. Test data MUST be synthetic. For HTTP/UI and +The original five suites include in-memory SQLite and direct action calls. +Additional ORM, migration and HTTP suites use disposable file-backed databases. +They need no external credentials or service. Test data MUST be synthetic. For HTTP/UI and file-backed tests, use a disposable checkout, a temporary `data_dir`, loopback binding, and an isolated port. Avoid a live site's database or uploads. Keep test output in the approved locations in @@ -84,22 +96,51 @@ test output in the approved locations in ## Executable checks -The portable entry point discovers every `TestPrograms/**/*.test.wfl` suite: +The complete portable entry point recursively discovers regular `*.test.wfl` +files under `TestPrograms/`, `tests/tooling/`, `tests/integration/`, +`tests/runtime/`, and `examples/`, sorts each group, then runs the pinned Scribe suite: ```sh -python scripts/run_tests.py -python scripts/run_tests.py --include-scribe +wfl --execution-timeout 1200 scripts/run_tests.wfl +wfl scripts/run_tests.wfl --group application +wfl scripts/run_tests.wfl --group tooling +wfl scripts/run_tests.wfl --group integration +wfl scripts/run_tests.wfl --group examples +wfl scripts/run_tests.wfl --group runtime +wfl scripts/run_tests.wfl --group scribe ``` -`--wfl /absolute/path/to/wfl` selects the executable. `--timeout 120` sets the -maximum seconds per suite; 120 is the default. The runner executes suites -sequentially from the proper working directory, preserves interpreter output, -reports every suite's result, and exits nonzero if any suite fails or times out. -An empty suite set, missing interpreter, or missing requested Scribe source is -an error. There are no automatic retries. - -`--include-scribe` also executes the upstream `tests/scribe.test.wfl` from a -temporary copy of the pinned submodule, with a fresh `build/` for its fixtures. +The default executable is `current_executable`, the exact runtime that launched +the runner. `--wfl /absolute/path/to/wfl` selects another executable; a bare +program name uses native PATH lookup. The resolved absolute identity is passed +to every suite as `args[0]`. `--timeout 120` sets the +maximum subprocess wait per suite; 120 is the default. A suite's own WFL runtime +configuration can impose a shorter limit. The WFL invocation option +`--execution-timeout 1200` supplies a finite 20-minute budget for the complete +runner, including discovery and every suite. Place it before the script path. +It changes neither the 120-second suite wait nor the children's own runtime +limits. Without this explicit option, `scripts/.wflcfg` supplies the runtime's +300-second configuration maximum, which is insufficient for the complete +Linux run. Setting `timeout_seconds` to zero does not disable that bound. +The CI job's longer timeout also includes provisioning and cleanup. +The runner executes suites +sequentially from the proper working directory, preserves stdout and stderr +contents in its output (stderr has a label), +reports suite results while the whole-run budget remains, and exits nonzero if +any suite fails or times out. Exhausting the whole-run budget stops the command +with a nonzero exit and owned-process cleanup. +An empty selected group, missing interpreter, invalid timeout, or missing +requested Scribe source is an error before suites start. Suite failure and +timeout exit 1; setup/cleanup failure exits 2. There are no automatic retries. +The runner owns child process trees, including on hard timeout. All temporary +fixtures remain under ignored `target/test-artifacts/` and are removed after +their suites. A terminated host/CI container can leave disposable files; it +does not authorize reuse of a live site's data. Helpers do not end in `.test.wfl`. + +The default command executes upstream `tests/scribe.test.wfl` from a temporary +copy of the pinned submodule, excluding `.git`, existing `build/`, and caches, +with a fresh `build/` for its fixtures. `--include-scribe` remains a compatibility +option to add Scribe to a focused group. This avoids writing test output into the dependency checkout. Run it for Scribe pin changes and changes affecting Scribe integration. CMS-specific Scribe regressions remain in `TestPrograms/scribe.test.wfl` even when upstream tests @@ -109,14 +150,15 @@ The HTTP port configuration checks start disposable Scriptorium processes and exercise `/install` using synthetic temporary databases: ```sh -python -m unittest discover -s tests/integration -v +wfl scripts/run_tests.wfl --group integration ``` -These checks require WFL on `PATH` (or `WFL_EXECUTABLE` set to its executable) -and an available loopback port 8080. They check an explicitly configured port, -the default when the setting or file is absent, and startup URLs. They do not -exercise installer submission or other complete CMS journeys. The WFL tests -workflow runs them inside its disposable nightly container after the WFL suites. +These checks use the selected WFL executable and require an available loopback +port 8080 for default-port cases. They preserve configured/default port and +startup URL checks and extend real installer, auth, content, users, media, +throttling and restore workflows. See [the integration suite](tests/integration/README.md) +for individual scenarios and fixture ownership. No environment-variable +interpreter override is needed; use the shared `--wfl` option. | Existing suite | Direct command from the repository root | What it currently exercises | |---|---|---| @@ -131,16 +173,18 @@ upgrade. The auth suite does not test the login router or complete role matrix. The render suite tests path selection; it does not render every theme template through HTTP. Keep these distinctions in PR descriptions. -Repository tooling checks run independently of WFL: +Repository tooling checks use the same WFL runner: ```sh python scripts/check_repo_hygiene.py -python -m unittest discover -s tests/tooling -v +wfl scripts/run_tests.wfl --group tooling ``` The tooling suites check the validation and runner failure paths. They do not substitute for the application suites. Application tests stay in the existing -`TestPrograms/` layout; Python tooling tests live in `tests/tooling/`. +`TestPrograms/` layout; WFL tooling tests live in `tests/tooling/` and HTTP tests +in `tests/integration/`. The [conversion inventory](docs/wfl-test-inventory.md) +maps all eight original runner requirements and 28 hygiene cases. ## CMS boundaries and critical journeys @@ -210,22 +254,37 @@ keyboard behavior, or data integrity. [Governance](.github/workflows/governance.yml) runs repository hygiene and tooling checks on Blacksmith Linux and GitHub-hosted Windows. -[WFL tests](.github/workflows/wfl-tests.yml) runs the five application suites and the pinned Scribe suite via -`python3 scripts/run_tests.py --include-scribe` on +[WFL tests](.github/workflows/wfl-tests.yml) runs the complete WFL suite via +`wfl --execution-timeout 1200 scripts/run_tests.wfl` on `blacksmith-2vcpu-ubuntu-2404`. Both workflows run for pushes and pull requests to `main` and support manual dispatch. WFL tests pulls `bsbyrdwfl/wfl:nightly` from Docker Hub for every run, resolves the image digest, and uses that immutable image for that run's runtime checks. -Python is installed in the disposable test container and the source checkout is -mounted read-only. The job summary records the resolved image digest, +Python and Git are installed only for the hygiene checker subject. No Python +test runner or test implementation executes. The source checkout is mounted +read-only and copied into the writable disposable container for fixtures. +The job summary records the resolved image digest, `wfl --version`, and tested Scriptorium and Scribe revisions. The nightly tag is moving: retain the run URL and digest with PR evidence so a later nightly does not obscure which runtime was tested. The nightly workflow is not a declaration that every nightly, platform, or production configuration is supported. -Automated HTTP coverage is limited to startup port configuration and serving -the installer form. Complete HTTP/browser journeys remain unautomated. The +Governance provisions the latest official nightly publication on Linux and +Windows from WFL's canonical download CDN. A unique manifest request key avoids +old rolling-pointer cache entries. It resolves the publication record to an +immutable versioned asset and verifies its immutable SHA256 sidecar and +executable version. The daily GitHub release is an immutable mirror that can +remain older after another publication on the same day. The job records the +WFL source revision, asset URL, SHA256 and runtime version before running WFL +tooling regressions. The runner requires WFL's owned-process cwd/timeout/full-result/close +API, explicit exit status and `current_executable`; the HTTP/ORM suites also need +the redirect and transaction remedies described in [runtime review](docs/runtime-review.md). +A source-built candidate passing locally does not establish a published nightly +pass; final immutable nightly and Windows/Linux remote results remain merge gates. + +Automated HTTP suites cover application journeys; browser interaction and +accessibility coverage remain separate requirements. The scheduled Scribe updater only proposes dependency changes; its successful run alone is not runtime test evidence. @@ -263,7 +322,7 @@ remain a separate adoption item below. | Gap | Required next step and trigger | |---|---| -| HTTP coverage is limited to port configuration; no browser automation | Add real-boundary regression coverage with each affected behavior change; plan coverage of all critical journeys before the next production release. | +| No browser automation or full accessibility evidence | Retain real HTTP boundary regressions and add browser checks for changed interactions; verify critical journeys before production releases. | | No declared compatibility matrix or release-candidate workflow | Define supported runtime/platform/configuration tuples and retain candidate results before the next production release. | | No coverage measurement, performance budgets, or scheduled extended tests | Establish baselines and risk-based targets before claiming those properties; review at the next profile review. | | Host protection settings are external | Maintainer verifies required checks and review rules on GitHub at adoption and after workflow changes. | diff --git a/tests/fixtures/legacy-database.wfl b/tests/fixtures/legacy-database.wfl new file mode 100644 index 0000000..f11896c --- /dev/null +++ b/tests/fixtures/legacy-database.wfl @@ -0,0 +1,28 @@ +include from "../../app/migrations.wfl" + +// Literal pre-ORM DDL from the persistence inventory. These expectations do not +// use the schema compiler or current application models to create their input. +define action called legacy_fixture_statements with parameters with_csrf: + store sessions_sql as "CREATE TABLE sessions (id TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL)" + check if with_csrf: + change sessions_sql to "CREATE TABLE sessions (id TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL, csrf_token TEXT NOT NULL DEFAULT '')" + end check + return ["CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'author', created_at TEXT DEFAULT (datetime('now')))", sessions_sql, "CREATE TABLE posts (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT UNIQUE NOT NULL, title TEXT NOT NULL, body_markdown TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'draft', author_id INTEGER, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')))", "CREATE TABLE pages (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT UNIQUE NOT NULL, title TEXT NOT NULL, body_markdown TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'draft', author_id INTEGER, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')))", "CREATE TABLE settings (skey TEXT PRIMARY KEY, svalue TEXT NOT NULL DEFAULT '')", "CREATE TABLE media (id INTEGER PRIMARY KEY AUTOINCREMENT, filename TEXT UNIQUE NOT NULL, original_name TEXT NOT NULL DEFAULT '', content_type TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0, uploader_id INTEGER, created_at TEXT DEFAULT (datetime('now')))", "CREATE TABLE login_attempts (id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, attempted_at TEXT DEFAULT (datetime('now')))", "CREATE INDEX idx_login_attempts_ip_time ON login_attempts (ip, attempted_at)"] +end action + +define action called legacy_fixture_path: + store artifact_root as path_join of current_directory and "target/test-artifacts/legacy-migrations" + call makedirs with artifact_root + store artifact_id as generate_uuid + return path_join of artifact_root and (artifact_id with ".db") +end action + +define action called legacy_fixture_remove with parameters conn and database_path: + close database conn + for each suffix_text in ["", "-wal", "-shm", "-journal"]: + store owned_path as database_path with suffix_text + check if file exists at owned_path: + delete file at owned_path + end check + end for +end action diff --git a/tests/fixtures/migration-registry.wfl b/tests/fixtures/migration-registry.wfl new file mode 100644 index 0000000..bd130f0 --- /dev/null +++ b/tests/fixtures/migration-registry.wfl @@ -0,0 +1,78 @@ +include from "../../lib/orm/migrations.wfl" + +define action called migration_fixture_table with parameters title_nullable and with_index: + create new OrmField as key_spec: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes + end + create new OrmField as title_spec: + field_name is "title" + nullable is title_nullable + end + store index_specs as [] + check if with_index: + create new OrmIndex as title_index: + index_name is "idx_items_title" + field_names is ["title"] + end + push with index_specs and title_index + end check + create new OrmModel as model_spec: + table_name is "items" + fields is [key_spec, title_spec] + indexes is index_specs + end + return orm_schema_table of model_spec +end action + +define action called migration_fixture_registry with parameters source_revision: + store first_table as migration_fixture_table of no and no + store indexed_table as migration_fixture_table of no and yes + store nullable_table as migration_fixture_table of yes and yes + create new OrmMigration as first_version: + migration_id is "20260920000001" + migration_name is "Create items" + up_steps is [(orm_migration_create_table of first_table)] + irreversible_reason is "the initial table may contain records" + managed_after is [first_table] + source_text is "fixture initial source\n" + end + create new OrmMigration as second_version: + migration_id is "20260920000002" + migration_name is "Index item titles" + up_steps is [(orm_migration_create_index of indexed_table and indexed_table.model_spec.indexes[0])] + down_steps is [(orm_migration_drop_index of indexed_table and indexed_table.model_spec.indexes[0])] + managed_before is [first_table] + managed_after is [indexed_table] + source_text is source_revision + end + create new OrmMigration as third_version: + migration_id is "20260920000003" + migration_name is "Allow missing titles" + up_steps is [(orm_migration_rebuild of indexed_table and nullable_table and ["id", "title"] and ["id", "title"])] + down_steps is [(orm_migration_rebuild of nullable_table and indexed_table and ["id", "title"] and ["id", "title"])] + managed_before is [indexed_table] + managed_after is [nullable_table] + source_text is "fixture rebuild source\n" + end + return [first_version, second_version, third_version] +end action + +define action called migration_fixture_path: + store artifact_root as path_join of current_directory and "target/test-artifacts/migrations" + call makedirs with artifact_root + store artifact_id as generate_uuid + return path_join of artifact_root and (artifact_id with ".db") +end action + +define action called migration_fixture_remove with parameters session and database_path: + call orm_close with session + for each suffix_text in ["", "-wal", "-shm", "-journal"]: + store owned_path as database_path with suffix_text + check if file exists at owned_path: + delete file at owned_path + end check + end for +end action diff --git a/tests/fixtures/migration-worker.wfl b/tests/fixtures/migration-worker.wfl new file mode 100644 index 0000000..afbbf0f --- /dev/null +++ b/tests/fixtures/migration-worker.wfl @@ -0,0 +1,58 @@ +include from "migration-registry.wfl" + +define action called migration_worker_signal with parameters signal_path: + open file at signal_path for writing as signal_file + try: + wait for write content "ready" into signal_file + finally: + close file signal_file + end try +end action + +store database_path as args[0] +store worker_mode as args[1] +store ready_path as args[2] +store registry as migration_fixture_registry of "fixture index source\n" +store session as orm_open of database_path +try: + check if worker_mode is equal to "apply": + call migration_worker_signal with ready_path + store migrated as orm_migrate of session and registry and args[3] + display "applied " with (length of migrated.applied) + otherwise: + store conn as session.connection + in transaction on conn for schema changes: + check if worker_mode is equal to "hold": + store created as orm_sql_execute of session and "CREATE TABLE abandoned_schema(value TEXT)" and [] + store pending_row as orm_sql_execute of session and "INSERT INTO items(title) VALUES('uncommitted')" and [] + store pending_event as orm_sql_execute of session and "INSERT INTO _orm_migration_events(migration_id,checksum,event_kind) VALUES('uncommitted','uncommitted','apply')" and [] + otherwise: + // A controlled competing owner advances the same immutable + // steps and ledger while another process starts its request. + count from 2 to 3: + store version_spec as registry[count minus 1] + for each migration_step in version_spec.up_steps: + store stepped as orm_migration_run_step of session and migration_step + end for + store verified as orm_migration_schema_check of session and registry and count + store recorded as orm_migration_record of session and version_spec and count and "apply" + end count + end check + call migration_worker_signal with ready_path + store release_path as args[3] + store released as no + count from 1 to 300: + check if file exists at release_path: + change released to yes + break + end check + wait for 100 milliseconds + end count + check if released is equal to no: + call raise_error with "Fixture owner was not released before its 30-second bound" + end check + end transaction + end check +finally: + call orm_close with session +end try diff --git a/tests/fixtures/orm-model.wfl b/tests/fixtures/orm-model.wfl new file mode 100644 index 0000000..94138e8 --- /dev/null +++ b/tests/fixtures/orm-model.wfl @@ -0,0 +1,46 @@ +include from "../../lib/orm/writes.wfl" + +define action called fixture_entries_model: + create new OrmField as key_spec: + field_name is "id" + value_type is "identity" + primary_key is yes + generated is yes + end + create new OrmField as slug_spec: + field_name is "slug" + unique_value is yes + end + create new OrmField as note_spec: + field_name is "note" + nullable is yes + end + create new OrmField as enabled_spec: + field_name is "enabled" + value_type is "boolean" + has_default is yes + default_value is no + end + create new OrmField as score_spec: + field_name is "score" + value_type is "integer" + has_default is yes + default_value is 7 + end + create new OrmModel as entries_model: + table_name is "entries" + fields is [key_spec and slug_spec and note_spec and enabled_spec and score_spec] + end + return entries_model +end action + +define action called fixture_reset_entries with parameters session: + call orm_sql_execute with session and "DROP TABLE IF EXISTS entries" and [] + call orm_sql_execute with session and "CREATE TABLE entries (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT NOT NULL UNIQUE, note TEXT, enabled INTEGER NOT NULL DEFAULT 0 CHECK(enabled IN (0,1)), score INTEGER NOT NULL DEFAULT 7)" and [] +end action + +define action called fixture_entry with parameters session and model_spec and slug_value: + store draft as orm_record of model_spec + call orm_set with draft and "slug" and slug_value + return orm_save of session and draft +end action diff --git a/tests/integration/.wflcfg b/tests/integration/.wflcfg new file mode 100644 index 0000000..c799c19 --- /dev/null +++ b/tests/integration/.wflcfg @@ -0,0 +1,9 @@ +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true +timeout_seconds = 120 +execution_logging = false +logging_enabled = false +debug_report_enabled = false +web_server_bind_address = 127.0.0.1 +max_buffer_size_bytes = 4194304 diff --git a/tests/integration/EVIDENCE.md b/tests/integration/EVIDENCE.md new file mode 100644 index 0000000..5968374 --- /dev/null +++ b/tests/integration/EVIDENCE.md @@ -0,0 +1,139 @@ +# HTTP integration evidence + +## Baseline characterization, 2026-09-20 + +Executed on Windows with WFL reporting `26.9.12`, built from combined upstream +revision `ae5395d9bd215d0d9fa1c039e666f03c8897cf88`. Executable SHA-256: +`76F612CA1BBAF4484B2CC2BDA86D876588BC765DE9C38B7DAA8951BAD143B911`. + +Application source baseline: Scriptorium +`4ec5c88d9e4ae27041599ad8293a28130b56fbf2`, extracted into ignored +`target/http-baseline-4ec5c88`. Both that revision and the current checkout pin +Scribe `93d62af5a6ed6c3ce257ef888107fc3ca1e2dc1d`. Git archive provisioned the +baseline source; WFL created every actual site, configuration, database, upload, +request, assertion and cleanup operation. + +Each suite used this command shape from `G:/repos/Scriptorium`: + +```text +G:/repos/wfl/target/codex-worktrees/integration/target/release/wfl.exe --test tests/integration/.test.wfl G:/repos/wfl/target/codex-worktrees/integration/target/release/wfl.exe G:/repos/Scriptorium/target/http-baseline-4ec5c88 +``` + +| Suite | Passed | Retained local log | +|---|---:|---| +| server-port | 3/3 | `target/http-baseline-ports.log` | +| authentication | 2/2 | `target/http-baseline-authentication.log` | +| content-users | 2/2 | `target/http-baseline-content-users.log` | +| media | 2/2 | `target/http-baseline-media.log` | +| throttle | 1/1 | `target/http-baseline-throttle-green.log` | +| recovery | 1/1 | `target/http-baseline-recovery.log` | + +Total: **11/11**. The media case exercises both configured and legacy storage +layouts. No assertions were removed or relaxed to obtain these results. +`target/test-artifacts/http` contained zero residual fixture directories after +the runs. Each WFL stop helper asserts its owned process is no longer running. + +The first throttle run failed because the test reused `login_cookie` outside +the `try` scope that declared it. Moving the cookie/token declarations to the +test's enclosing scope fixed that fixture defect; its original log remains +`target/http-baseline-throttle.log`. This is not a production defect or Red +evidence for the application. Earlier parse/smoke attempts using separate, +incomplete prerequisite binaries could not resolve all new APIs and are not +counted as integration evidence. + +The interpreter emits advisory undefined-action warnings for actions supplied +by the included helper and unused-variable warnings for values read by test +assertions. They remain visible in the logs; every suite exits zero and reports +the passing counts above. A separate OS process-list audit was unavailable +because Windows denied the read; cleanup evidence is the owned-process checks +and empty fixture directory, not a claim of an independent OS-wide audit. + +## Updated application candidate + +The same six suites ran against the updated application working tree on +2026-09-20 with the same combined runtime, omitting the baseline-source argument. +All **11/11** passed on their first candidate run: + +| Suite | Passed | Retained local log | +|---|---:|---| +| server-port | 3/3 | `target/http-candidate-ports.log` | +| authentication | 2/2 | `target/http-candidate-authentication.log` | +| content-users | 2/2 | `target/http-candidate-content-users.log` | +| media | 2/2 | `target/http-candidate-media.log` | +| throttle | 1/1 | `target/http-candidate-throttle.log` | +| recovery | 1/1 | `target/http-candidate-recovery.log` | + +These runs cover the first integrated ORM/migration adapters before the later +independent nullable-key and write-metadata compatibility fixes. Those fixes +have separate Red/Green tests in `TestPrograms/db-review-contracts.test.wfl`. +The application was still an uncommitted working tree, so this evidence is not +a claim that an immutable final application commit has passed. The complete +runner must also test the final proposed commit. Linux/nightly container +evidence and its immutable image digest remain separate required checks; +local Windows results do not establish those results. + +The original three Python port cases map directly to the three WFL port cases, +including installer status/content type/form/CSRF, configured/legacy/default +ports and startup messages. The Python file was removed only after its WFL +replacement passed against both baseline and the updated application. + +## Independent review + +An independent agent reviewed fixture process ownership and the stopped-site +backup/restore protocol. It found no blocking source issue: unique owned +directories, cleanup after failures, reaping before copying all data/sidecars, +and HTTP plus SQLite checks after restore. This is technical review, not +Maintainer approval or release acceptance. + +## Legacy extension and theme upgrade boundary + +`legacy-upgrade.test.wfl` passed **1/1** on 2026-09-20 with the same combined +runtime against the working candidate after the compatibility fixes. Log: +`target/http-candidate-legacy-upgrade.log`. This adds a twelfth HTTP case. + +WFL creates the original seven-table schema without `sessions.csrf_token` or a +migration ledger, a synthetic Argon2 administrator, an installed flag, published +content, a live legacy session and an extension-owned payload. It copies a WFL +extension into the disposable site's documented seam and generates a configured +custom theme. Actual boot adopts the schema and creates extension boot markers. +HTTP proves both installer methods remain locked, the extension owns `/` before +stock dispatch, the custom theme renders extension and stock post routes, the +retained user can log in, and a legacy empty CSRF token cannot mutate settings. + +A stopped/reaped restart preserves identical ledger rows/checksums/timestamps, +retained extension payload and managed rows; only the extension's expected boot +count increments. Integrity and foreign-key checks pass. The fixture has no +requirement for the ignored Git-archive baseline and no non-WFL test logic. + +Independent source review found no fixture ownership or application-boundary +issue. It requested ordered ledger snapshots so map iteration order cannot cause +false failures; the test now serializes ordered lists of explicit fields. + +## Portable request-body rejection, 2026-09-20 + +The original 16 MiB upload received an HTTP transport error on Linux in +[Red run 35507634634, job 106070018120](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35507634634/job/106070018120) +at head `07e8adcb7785c168c37fd56cc35e88492809a820`. The same media suite passed +2/2 on Windows. WFL rejects an oversized advertised `Content-Length` before +reading the body; closing an unread large upload can reset the connection before +the client observes 413. This was a fixture portability assumption, not evidence +that the upload was accepted. The test did not accept the transport error as a +passing result. + +The corrected fixture asserts that the shipped configuration and normal fixture +both retain the 10 MiB ceiling, then sets only its disposable site to 1 KiB. It +sends a real 2 KiB multipart file and requires **413, zero media rows, zero upload +files, and a subsequent authenticated GET returning 200**. Existing configured +and legacy upload workflows remain covered. No retry, skip, transport-error +fallback, application change or runtime change was introduced. This does not +claim that eager 16 MiB HTTP/1 uploads always expose a 413 response. + +[Green run 35508309315, job 106071763602](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35508309315/job/106071763602) +ran the actual media suite on Blacksmith Linux at +`147b15c88c5fbd29e4826a32a0db5daf747ae59d`: **2/2 passed**. Its freshly resolved +official runtime was WFL **26.9.14**, source +`8d82d785ea59300834de1c48b04a7ba0e187a1cd`, image +`bsbyrdwfl/wfl@sha256:8498abec67995274cb4d6cc2ee9580be11f70e15af20497bb62e51d4b2058e3c`. +Scribe remained `93d62af5a6ed6c3ce257ef888107fc3ca1e2dc1d`. The same revised +suite passed **2/2** using the official Windows 26.9.14 runtime; retained local +log: `target/media-probe/media-small-official-windows.log`. diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000..7a1ed65 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,70 @@ +# Real HTTP integration suites + +Run from the repository root with the candidate executable as the first WFL +program argument, for example: + +```text +wfl --test tests/integration/server-port.test.wfl /absolute/path/to/wfl +wfl --test tests/integration/authentication.test.wfl /absolute/path/to/wfl +wfl --test tests/integration/content-users.test.wfl /absolute/path/to/wfl +wfl --test tests/integration/media.test.wfl /absolute/path/to/wfl +wfl --test tests/integration/throttle.test.wfl /absolute/path/to/wfl +wfl --test tests/integration/recovery.test.wfl /absolute/path/to/wfl +wfl --test tests/integration/legacy-upgrade.test.wfl /absolute/path/to/wfl +``` + +The runtime must support native per-child working directories, owned process +completion results, `current_executable`, explicit application errors, and per-request redirect +control. The suite driver and all children use the supplied executable. Without +the argument, the helper uses `current_executable` to retain its own runtime. +Run sequentially; the two +default-port cases require loopback port 8080 to be available. A bind failure is +a failed prerequisite; no existing process is stopped or reused. + +An optional second program argument selects an extracted application source +baseline for before/after characterization. Artifacts and the WFL request helper +still belong to this checkout; the source baseline supplies only application +code/templates. The pinned Scribe entry is copied from this checkout, so record +and verify its unchanged revision when comparing an older application baseline. + +All fixture creation, HTTP requests, assertions, SQL inspection, process +ownership, and cleanup are WFL. Each site has a UUID directory beneath +`target/test-artifacts/http`. It copies application/library WFL and theme/admin +HTML plus the pinned Scribe entry point. It never copies checkout databases, +uploads, local configuration, Git metadata, or another site's state. The absent +configuration case inherits a loopback-only runtime configuration from its +disposable parent. The generated site runs in its own working directory. + +The helper owns the direct site process, waits up to 20 seconds for its startup +message, captures early-exit stdout/stderr, and closes it in `finally`. Each +request uses a separate WFL helper with a five-second runtime budget and a +six-second owned-process deadline. It returns the actual redirect response and +all response-header values. Tests manage synthetic cookies explicitly. + +| Suite | Real boundaries asserted | +|---|---| +| `server-port.test.wfl` | Original Python cases: configured ephemeral port, missing setting, absent config; exact startup URLs, installer status/content type/form/CSRF. | +| `authentication.test.wfl` | Installer CSRF and input rejection without mutation, first admin/session creation, setup locking, login response cookie flags, invalid credentials, CSRF, logout methods, and expired sessions. | +| `content-users.test.wfl` | Draft/publication, escaped title and rendered Markdown, pagination, page navigation/update/delete, mutation methods, CSRF, author ownership, admin user/settings boundaries, password update, account removal and revoked access. | +| `media.test.wfl` | Configured and legacy storage, generated safe names, byte retrieval, missing/wrong CSRF, extension/empty/malformed/oversized rejection, author denial, method enforcement, database/file consistency on deletion. | +| `throttle.test.wfl` | Exactly ten failed credentials, next request blocked, persistence across restart, expired-window recovery and successful-login clearing. | +| `recovery.test.wfl` | Stopped-site database-plus-upload backup, restart, actual restore after a later change, HTTP content/media and login, SQLite integrity and foreign-key checks. | +| `legacy-upgrade.test.wfl` | Pre-CSRF installed-site adoption and restart; existing admin/session and published content, installer lock, custom extension boot/route and configured theme, unchanged migration history and extension data. | + +SQL reads inspect the synthetic site independently; installation, account and +content creation use the real HTTP routes. Outside the historical-upgrade +fixture, SQL writes only simulate expired +sessions and elapsed throttle windows. Upload payloads are synthetic text with +image extensions because the current CMS validates extensions, not image +decoding. These checks do not claim browser accessibility or image-content +inspection. Migration upgrade/crash-recovery coverage belongs to the migration +suites; the HTTP recovery test restores a matching stopped-site data backup. +The legacy-upgrade fixture creates the frozen historical schema and synthetic +persisted rows directly in WFL, then verifies adoption through actual application +boot and HTTP routes. It requires the updated application and does not run +against the optional original-source baseline. + +The three original Python port cases have passed through their WFL replacements, +and the Python file has been removed. Its original source remains in Git at +`4ec5c88d9e4ae27041599ad8293a28130b56fbf2`. Before/after evidence is recorded in +[EVIDENCE.md](EVIDENCE.md); parsing alone is not passing integration proof. diff --git a/tests/integration/authentication.test.wfl b/tests/integration/authentication.test.wfl new file mode 100644 index 0000000..06cbeaa --- /dev/null +++ b/tests/integration/authentication.test.wfl @@ -0,0 +1,96 @@ +include from "fixtures/http-helpers.wfl" + +describe "Installation and authenticated HTTP sessions": + test "installer rejects CSRF without mutation and locks after successful setup": + store site_spec as http_new_site of "configured" + try: + store running_site as http_launch of site_spec + try: + store first_visit as http_get of site_spec and "/" and "" + call http_redirect with first_visit and "/install" + store installer as http_get of site_spec and "/install" and "" + store csrf_cookie as http_cookie of installer and "csrf" + store invalid_form as "site_title=Rejected&username=admin&password=Synthetic-pass-2026!&password_confirm=Synthetic-pass-2026!&csrf_token=wrong" + store invalid_install as http_post of site_spec and "/install" and csrf_cookie and invalid_form + expect invalid_install["status"] to equal 200 + expect invalid_install["body"] to contain "form had expired" + expect http_total of site_spec and "users" to equal 0 + expect http_total of site_spec and "sessions" to equal 0 + store install_token as http_csrf of installer + store invalid_password as http_post of site_spec and "/install" and csrf_cookie and ("site_title=Rejected&username=admin&password=Synthetic-pass-2026!&password_confirm=Different&csrf_token=" with install_token) + expect invalid_password["status"] to equal 200 + expect invalid_password["body"] to contain "Passwords do not match" + expect http_total of site_spec and "users" to equal 0 + store admin_cookie as http_install of site_spec + expect http_total of site_spec and "users" to equal 1 + expect http_total of site_spec and "sessions" to equal 1 + store dashboard as http_get of site_spec and "/admin" and admin_cookie + expect dashboard["status"] to equal 200 + expect dashboard["body"] to contain "Synthetic Journal" + store late_get as http_get of site_spec and "/install" and "" + call http_redirect with late_get and "/" + store late_post as http_post of site_spec and "/install" and csrf_cookie and invalid_form + call http_redirect with late_post and "/" + expect http_total of site_spec and "users" to equal 1 + store public_home as http_get of site_spec and "/" and "" + expect public_home["body"] to contain "Synthetic Journal" + expect public_home["body"] contains "Rejected" to be no + finally: + call http_stop with running_site + end try + finally: + call http_destroy with site_spec + end try + end test + + test "login cookies CSRF logout and server-side expiry cross the real router": + store site_spec as http_new_site of "configured" + try: + store running_site as http_launch of site_spec + try: + store installed_cookie as http_install of site_spec + store login_form as http_get of site_spec and "/admin/login" and "" + store login_cookie as http_cookie of login_form and "csrf" + store login_token as http_csrf of login_form + store missing_csrf as http_post of site_spec and "/admin/login" and "" and "username=admin&password=Synthetic-pass-2026!" + expect missing_csrf["status"] to equal 200 + expect missing_csrf["body"] to contain "form had expired" + expect http_total of site_spec and "login_attempts" to equal 0 + store wrong_login as http_post of site_spec and "/admin/login" and login_cookie and ("username=admin&password=wrong&csrf_token=" with login_token) + expect wrong_login["status"] to equal 200 + expect wrong_login["body"] to contain "Wrong username or password" + expect http_total of site_spec and "login_attempts" to equal 1 + store authenticated as http_post of site_spec and "/admin/login" and login_cookie and ("username=admin&password=Synthetic-pass-2026!&csrf_token=" with login_token) + call http_redirect with authenticated and "/admin" + store response_headers as authenticated["headers"] + expect response_headers["set-cookie"] to contain "HttpOnly" + expect response_headers["set-cookie"] to contain "SameSite=Lax" + expect response_headers["set-cookie"] to contain "Path=/" + store session_cookie as http_cookie of authenticated and "sid" + expect session_cookie is not equal to installed_cookie to be yes + expect http_total of site_spec and "login_attempts" to equal 0 + store dashboard as http_get of site_spec and "/admin" and session_cookie + store session_token as http_csrf of dashboard + store get_logout as http_get of site_spec and "/admin/logout" and session_cookie + call http_redirect with get_logout and "/admin" + store missing_logout as http_post of site_spec and "/admin/logout" and session_cookie and "" + expect missing_logout["status"] to equal 403 + store still_logged_in as http_get of site_spec and "/admin" and session_cookie + expect still_logged_in["status"] to equal 200 + store logout_reply as http_post of site_spec and "/admin/logout" and session_cookie and ("csrf_token=" with session_token) + call http_redirect with logout_reply and "/admin/login" + store logout_headers as logout_reply["headers"] + expect logout_headers["set-cookie"] to contain "Max-Age=0" + store after_logout as http_get of site_spec and "/admin" and session_cookie + call http_redirect with after_logout and "/admin/login" + store expired as http_execute of site_spec and "UPDATE sessions SET expires_at = '2000-01-01 00:00:00'" + store after_expiry as http_get of site_spec and "/admin" and installed_cookie + call http_redirect with after_expiry and "/admin/login" + finally: + call http_stop with running_site + end try + finally: + call http_destroy with site_spec + end try + end test +end describe diff --git a/tests/integration/content-users.test.wfl b/tests/integration/content-users.test.wfl new file mode 100644 index 0000000..6248fad --- /dev/null +++ b/tests/integration/content-users.test.wfl @@ -0,0 +1,142 @@ +include from "fixtures/http-helpers.wfl" + +describe "Content roles and settings through HTTP": + test "draft publication escaping pagination page navigation and delete methods": + store site_spec as http_new_site of "configured" + try: + store running_site as http_launch of site_spec + try: + store admin_cookie as http_install of site_spec + store csrf_token as http_session_token of site_spec and admin_cookie + store rejected as http_post of site_spec and "/admin/posts" and admin_cookie and "title=Rejected&slug=rejected&status=published&csrf_token=wrong" + expect rejected["status"] to equal 403 + expect http_total of site_spec and "posts" to equal 0 + store draft_body as "title=Unsafe+%3Cscript%3Ealert%281%29%3C%2Fscript%3E&slug=first-post&body_markdown=Hello+**reader**&status=draft&csrf_token=" with csrf_token + store created as http_post of site_spec and "/admin/posts" and admin_cookie and draft_body + call http_redirect with created and "/admin/posts" + store draft as http_get of site_spec and "/post/first-post" and "" + expect draft["status"] to equal 404 + store post_row as http_first_row of site_spec and "SELECT id, title, status FROM posts WHERE slug = 'first-post'" + expect post_row["status"] to equal "draft" + store edit_path as "/admin/posts/" with post_row["id"] + store published_body as replace "status=draft" with "status=published" in draft_body + store updated as http_post of site_spec and edit_path and admin_cookie and published_body + call http_redirect with updated and "/admin/posts" + store public_post as http_get of site_spec and "/post/first-post" and "" + expect public_post["status"] to equal 200 + expect public_post["body"] to contain "<script>" + expect public_post["body"] contains "" to be no + expect public_post["body"] to contain "reader" + store second_post as http_post of site_spec and "/admin/posts" and admin_cookie and ("title=Second&slug=second-post&body_markdown=Second+body&status=published&csrf_token=" with csrf_token) + call http_redirect with second_post and "/admin/posts" + store settings_reply as http_post of site_spec and "/admin/settings" and admin_cookie and ("site_title=Changed+Journal&site_tagline=New+tagline&posts_per_page=1&csrf_token=" with csrf_token) + call http_redirect with settings_reply and "/admin/settings" + store home as http_get of site_spec and "/" and "" + expect home["status"] to equal 200 + expect home["body"] to contain "Changed Journal" + expect home["body"] to contain "/blog/page/2" + store next_page as http_get of site_spec and "/blog/page/2" and "" + expect next_page["status"] to equal 200 + store created_page as http_post of site_spec and "/admin/pages" and admin_cookie and ("title=About+Journal&slug=about&body_markdown=About+our+site&status=published&csrf_token=" with csrf_token) + call http_redirect with created_page and "/admin/pages" + store public_page as http_get of site_spec and "/page/about" and "" + expect public_page["status"] to equal 200 + expect public_page["body"] to contain "About our site" + store home_with_page as http_get of site_spec and "/" and "" + expect home_with_page["body"] to contain "/page/about" + store page_row as http_first_row of site_spec and "SELECT id FROM pages WHERE slug = 'about'" + store page_path as "/admin/pages/" with page_row["id"] + store page_updated as http_post of site_spec and page_path and admin_cookie and ("title=Revised+About&slug=about&body_markdown=Revised+page&status=published&csrf_token=" with csrf_token) + call http_redirect with page_updated and "/admin/pages" + store revised_page as http_get of site_spec and "/page/about" and "" + expect revised_page["body"] to contain "Revised page" + for each delete_path in [(edit_path with "/delete"), (page_path with "/delete")]: + store get_delete as http_get of site_spec and delete_path and admin_cookie + expect get_delete["status"] to equal 405 + store csrf_delete as http_post of site_spec and delete_path and admin_cookie and "csrf_token=wrong" + expect csrf_delete["status"] to equal 403 + end for + expect http_total of site_spec and "posts" to equal 2 + expect http_total of site_spec and "pages" to equal 1 + store deleted_post as http_post of site_spec and (edit_path with "/delete") and admin_cookie and ("csrf_token=" with csrf_token) + call http_redirect with deleted_post and "/admin/posts" + store deleted_page as http_post of site_spec and (page_path with "/delete") and admin_cookie and ("csrf_token=" with csrf_token) + call http_redirect with deleted_page and "/admin/pages" + store gone_post as http_get of site_spec and "/post/first-post" and "" + store gone_page as http_get of site_spec and "/page/about" and "" + expect gone_post["status"] to equal 404 + expect gone_page["status"] to equal 404 + finally: + call http_stop with running_site + end try + finally: + call http_destroy with site_spec + end try + end test + + test "authors own their content and admin controls users and settings": + store site_spec as http_new_site of "configured" + try: + store running_site as http_launch of site_spec + try: + store admin_cookie as http_install of site_spec + store admin_token as http_session_token of site_spec and admin_cookie + store author_created as http_post of site_spec and "/admin/users" and admin_cookie and ("username=author&password=Author-pass-2026!&role=author&csrf_token=" with admin_token) + call http_redirect with author_created and "/admin/users" + store author_row as http_first_row of site_spec and "SELECT id, role FROM users WHERE username = 'author'" + expect author_row["role"] to equal "author" + store author_cookie as http_login of site_spec and "author" and "Author-pass-2026!" + store author_token as http_session_token of site_spec and author_cookie + for each admin_route in ["/admin/users", "/admin/users/new", "/admin/settings"]: + store forbidden_get as http_get of site_spec and admin_route and author_cookie + expect forbidden_get["status"] to equal 403 + end for + store forbidden_settings as http_post of site_spec and "/admin/settings" and author_cookie and ("site_title=Unauthorized&csrf_token=" with author_token) + expect forbidden_settings["status"] to equal 403 + store forbidden_user as http_post of site_spec and "/admin/users" and author_cookie and ("username=intruder&password=Not-created&role=admin&csrf_token=" with author_token) + expect forbidden_user["status"] to equal 403 + expect http_total of site_spec and "users" to equal 2 + store admin_post as http_post of site_spec and "/admin/posts" and admin_cookie and ("title=Admin+owned&slug=admin-owned&body_markdown=Retained&status=published&csrf_token=" with admin_token) + call http_redirect with admin_post and "/admin/posts" + store admin_post_row as http_first_row of site_spec and "SELECT id FROM posts WHERE slug = 'admin-owned'" + store admin_post_path as "/admin/posts/" with admin_post_row["id"] + store denied_edit as http_get of site_spec and (admin_post_path with "/edit") and author_cookie + expect denied_edit["status"] to equal 403 + store denied_update as http_post of site_spec and admin_post_path and author_cookie and ("title=Tampered&slug=admin-owned&status=published&csrf_token=" with author_token) + expect denied_update["status"] to equal 403 + store denied_delete as http_post of site_spec and (admin_post_path with "/delete") and author_cookie and ("csrf_token=" with author_token) + expect denied_delete["status"] to equal 403 + store unchanged as http_get of site_spec and "/post/admin-owned" and "" + expect unchanged["body"] to contain "Retained" + store own_post as http_post of site_spec and "/admin/posts" and author_cookie and ("title=Author+owned&slug=author-owned&body_markdown=Author+body&status=draft&csrf_token=" with author_token) + call http_redirect with own_post and "/admin/posts" + store own_row as http_first_row of site_spec and "SELECT id, author_id FROM posts WHERE slug = 'author-owned'" + expect own_row["author_id"] to equal author_row["id"] + store own_path as "/admin/posts/" with own_row["id"] + store own_updated as http_post of site_spec and own_path and author_cookie and ("title=Author+published&slug=author-owned&body_markdown=Published+by+author&status=published&csrf_token=" with author_token) + call http_redirect with own_updated and "/admin/posts" + store own_public as http_get of site_spec and "/post/author-owned" and "" + expect own_public["body"] to contain "Published by author" + store own_deleted as http_post of site_spec and (own_path with "/delete") and author_cookie and ("csrf_token=" with author_token) + call http_redirect with own_deleted and "/admin/posts" + store user_path as "/admin/users/" with author_row["id"] + store user_updated as http_post of site_spec and user_path and admin_cookie and ("role=author&password=Changed-pass-2026!&csrf_token=" with admin_token) + call http_redirect with user_updated and "/admin/users" + store changed_cookie as http_login of site_spec and "author" and "Changed-pass-2026!" + store self_delete as http_post of site_spec and "/admin/users/1/delete" and admin_cookie and ("csrf_token=" with admin_token) + expect self_delete["status"] to equal 400 + store get_user_delete as http_get of site_spec and (user_path with "/delete") and admin_cookie + expect get_user_delete["status"] to equal 405 + store user_deleted as http_post of site_spec and (user_path with "/delete") and admin_cookie and ("csrf_token=" with admin_token) + call http_redirect with user_deleted and "/admin/users" + expect http_total of site_spec and "users" to equal 1 + store revoked as http_get of site_spec and "/admin" and changed_cookie + call http_redirect with revoked and "/admin/login" + finally: + call http_stop with running_site + end try + finally: + call http_destroy with site_spec + end try + end test +end describe diff --git a/tests/integration/fixtures/.wflcfg b/tests/integration/fixtures/.wflcfg new file mode 100644 index 0000000..f0a958e --- /dev/null +++ b/tests/integration/fixtures/.wflcfg @@ -0,0 +1,6 @@ +timeout_seconds = 5 +execution_logging = false +logging_enabled = false +debug_report_enabled = false +kill_on_shutdown = true +web_server_bind_address = 127.0.0.1 diff --git a/tests/integration/fixtures/http-helpers.wfl b/tests/integration/fixtures/http-helpers.wfl new file mode 100644 index 0000000..d3bbd06 --- /dev/null +++ b/tests/integration/fixtures/http-helpers.wfl @@ -0,0 +1,311 @@ +// All fixture generation, process ownership, HTTP requests and assertions are +// WFL. Only synthetic files under this run's target/test-artifacts root change. +store http_repo_root as current_directory +store http_source_root as http_repo_root +store http_runtime as call current_executable +check if (length of args) is greater than 0: + change http_runtime to args[0] +end check +check if (length of args) is greater than 1: + change http_source_root to args[1] +end check +store http_fixture_root as path_join of http_repo_root and "tests/integration/fixtures" +store http_base_config as "web_server_bind_address = 127.0.0.1\ntimeout_seconds = 60\nexecution_logging = false\nlogging_enabled = false\ndebug_report_enabled = false\nkill_on_shutdown = true\nweb_server_max_body_size = 10485760\n" + +define action called http_write with parameters file_path and file_text: + call makedirs with (path_dirname of file_path) + open file at file_path for writing as output_file + wait for write content file_text into output_file + close file output_file +end action + +define action called http_copy_tree with parameters source_dir and target_dir: + call makedirs with target_dir + store entry_names as list_dir of source_dir + for each entry_name in entry_names: + check if entry_name is not equal to ".git" and entry_name is not equal to "uploads": + store source_path as path_join of source_dir and entry_name + store target_path as path_join of target_dir and entry_name + check if is_dir of source_path: + call http_copy_tree with source_path and target_path + otherwise: + check if entry_name ends with ".wfl" or entry_name ends with ".html": + call copy_file with source_path and target_path + end check + end check + end check + end for +end action + +define action called http_available_port with parameters wanted_port: + // Binding proves ownership before any request; occupied ports fail closed. + listen on port wanted_port as reservation_server + store reservation_text as "" with reservation_server + close server reservation_server + store address_parts as split reservation_text by ":" + return parse_json of address_parts[(length of address_parts) minus 1] +end action + +define action called http_new_site with parameters config_mode: + store run_id as generate_uuid + store run_root as path_join of http_repo_root and "target/test-artifacts/http" and run_id + store site_root as path_join of run_root and "site" + try: + call makedirs with site_root + call http_write with (path_join of run_root and ".wflcfg") and http_base_config + call copy_file with (path_join of http_source_root and "main.wfl") and (path_join of site_root and "main.wfl") + for each source_name in ["app", "admin", "themes", "lib/orm", "migrations"]: + store source_dir as path_join of http_source_root and source_name + check if is_dir of source_dir: + call http_copy_tree with source_dir and (path_join of site_root and source_name) + end check + end for + call makedirs with (path_join of site_root and "static") + store scribe_target as path_join of site_root and "lib/scribe/src" + call makedirs with scribe_target + call copy_file with (path_join of http_repo_root and "lib/scribe/src/scribe.wfl") and (path_join of scribe_target and "scribe.wfl") + store selected_port as 8080 + check if config_mode is equal to "configured" or config_mode is equal to "legacy": + change selected_port to http_available_port of 0 + expect selected_port is not equal to 8080 to be yes + otherwise: + store checked_port as http_available_port of 8080 + end check + check if config_mode is not equal to "absent": + store site_config as http_base_config + check if config_mode is not equal to "legacy": + change site_config to site_config with "data_dir = runtime-data\n" + end check + check if config_mode is equal to "configured" or config_mode is equal to "legacy": + change site_config to site_config with "web_server_port = " with selected_port with "\n" + end check + call http_write with (path_join of site_root and ".wflcfg") and site_config + end check + store database_path as path_join of site_root and "runtime-data/scriptorium.db" + store uploads_path as path_join of site_root and "runtime-data/uploads" + check if config_mode is equal to "absent" or config_mode is equal to "legacy": + change database_path to path_join of site_root and "scriptorium.db" + change uploads_path to path_join of site_root and "static/uploads" + end check + create map site_spec: + "root" is run_root + "site" is site_root + "port" is selected_port + "url" is "http://127.0.0.1:" with selected_port + "database" is database_path + "uploads" is uploads_path + end map + return site_spec + when error: + check if is_dir of run_root: + call remove_dir with run_root and yes + end check + call raise_error with error_message + end try +end action + +define action called http_session_token with parameters site_spec and session_cookie: + store dashboard as http_get of site_spec and "/admin" and session_cookie + expect dashboard["status"] to equal 200 + return http_csrf of dashboard +end action + +define action called http_first_row with parameters site_spec and sql_text: + store selected_rows as http_rows of site_spec and sql_text + expect length of selected_rows to equal 1 + return selected_rows[0] +end action + + +define action called http_launch with parameters site_spec: + store site_root as site_spec["site"] + store main_path as path_join of site_root and "main.wfl" + wait for spawn command http_runtime with arguments [main_path] in directory site_root as server_process + store startup_text as "" + store ready_seen as no + try: + count from 1 to 400: + wait for read output from process server_process as output_chunk + change startup_text to startup_text with output_chunk + check if startup_text contains "Scriptorium is running at ": + change ready_seen to yes + break + end check + check if (process server_process is running) is no: + wait for process server_process to complete with timeout 2 and read result as dead_server + display startup_text with dead_server["output"] with dead_server["error"] + call raise_error with "Disposable Scriptorium exited before readiness" + end check + wait for 50 milliseconds + end count + check if ready_seen is no: + display startup_text + call raise_error with "Disposable Scriptorium did not start within 20 seconds" + end check + when error: + close process server_process + call raise_error with error_message + end try + create map running_site: + "spec" is site_spec + "process" is server_process + "startup" is startup_text + end map + return running_site +end action + +define action called http_stop with parameters running_site: + store server_process as running_site["process"] + try: + check if process server_process is running: + wait for read output from process server_process as final_output + check if final_output contains "Request error": + display final_output + end check + otherwise: + wait for process server_process to complete with timeout 2 and read result as completed_server + display completed_server["output"] with completed_server["error"] + end check + finally: + close process server_process + end try + expect (process server_process is running) to equal no +end action + +define action called http_destroy with parameters site_spec: + call remove_dir with site_spec["root"] and yes +end action + +define action called http_request with parameters site_spec and route_path and verb_text and cookie_text and body_text and content_type_text: + store request_id as generate_uuid + store request_path as path_join of site_spec["root"] and (request_id with ".json") + create map request_spec: + "url" is site_spec["url"] with route_path + "method" is verb_text + "cookie" is cookie_text + "body" is body_text + "content_type" is content_type_text + end map + call http_write with request_path and (stringify_json of request_spec) + store request_program as path_join of http_fixture_root and "request.wfl" + wait for spawn command http_runtime with arguments [request_program, request_path] as request_process + try: + wait for process request_process to complete with timeout 6 and read result as completed_request + check if completed_request["exit_code"] is not equal to 0: + display completed_request["error"] + call raise_error with ("HTTP request failed for " with verb_text with " " with route_path) + end check + return parse_json of completed_request["output"] + finally: + close process request_process + delete file at request_path + end try +end action + +define action called http_get with parameters site_spec and route_path and cookie_text: + return http_request of site_spec and route_path and "GET" and cookie_text and "" and "application/x-www-form-urlencoded" +end action + +define action called http_post with parameters site_spec and route_path and cookie_text and body_text: + return http_request of site_spec and route_path and "POST" and cookie_text and body_text and "application/x-www-form-urlencoded" +end action + +define action called http_redirect with parameters reply and destination: + expect reply["status"] to equal 303 + store response_headers as reply["headers"] + expect response_headers["location"] to equal destination +end action + +define action called http_cookie with parameters reply and cookie_name: + store value_map as reply["header_values"] + expect value_map contains "set-cookie" to be yes + store cookie_values as [] + change cookie_values to value_map["set-cookie"] + for each cookie_value in cookie_values: + check if cookie_value starts with (cookie_name with "="): + store cookie_parts as split cookie_value by ";" + return cookie_parts[0] + end check + end for + call raise_error with ("Expected response cookie " with cookie_name) +end action + +define action called http_csrf with parameters reply: + store html_text as reply["body"] + store token_marker as "name=\"csrf_token\" value=\"" + store token_start as indexof of html_text and token_marker + expect token_start is greater than or equal to 0 to be yes + change token_start to token_start plus (length of token_marker) + store token_tail as substring of html_text and token_start and ((length of html_text) minus token_start) + store token_end as indexof of token_tail and "\"" + expect token_end is greater than 0 to be yes + return substring of token_tail and 0 and token_end +end action + +define action called http_install with parameters site_spec: + store installer as http_get of site_spec and "/install" and "" + expect installer["status"] to equal 200 + store install_token as http_csrf of installer + store install_cookie as http_cookie of installer and "csrf" + store install_form as "site_title=Synthetic+Journal&site_tagline=HTTP+fixtures&username=admin&password=Synthetic-pass-2026!&password_confirm=Synthetic-pass-2026!&csrf_token=" with install_token + store installed as http_post of site_spec and "/install" and install_cookie and install_form + call http_redirect with installed and "/admin" + return http_cookie of installed and "sid" +end action + +define action called http_login with parameters site_spec and username_text and password_text: + store login_form as http_get of site_spec and "/admin/login" and "" + store login_token as http_csrf of login_form + store login_cookie as http_cookie of login_form and "csrf" + store credentials as "username=" with username_text with "&password=" with password_text with "&csrf_token=" with login_token + store logged_in as http_post of site_spec and "/admin/login" and login_cookie and credentials + call http_redirect with logged_in and "/admin" + return http_cookie of logged_in and "sid" +end action + +define action called http_rows with parameters site_spec and sql_text: + store database_url as "sqlite://" with site_spec["database"] + open database at database_url as inspection_db + try: + return query inspection_db with sql_text + finally: + close database inspection_db + end try +end action + +define action called http_total with parameters site_spec and table_name: + store totals as http_rows of site_spec and ("SELECT COUNT(*) AS total FROM " with table_name) + store total_row as totals[0] + return total_row["total"] +end action + +define action called http_execute with parameters site_spec and sql_text: + // Synthetic fixture state only (expiry/window simulation), never app setup. + store database_url as "sqlite://" with site_spec["database"] + open database at database_url as inspection_db + try: + return execute inspection_db with sql_text + finally: + close database inspection_db + end try +end action + +define action called http_upload with parameters site_spec and cookie_text and csrf_text and filename_text and file_text: + store multipart_body as "--wfl-fixture-boundary\r\nContent-Disposition: form-data; name=\"csrf_token\"\r\n\r\n" with csrf_text with "\r\n--wfl-fixture-boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"" with filename_text with "\"\r\nContent-Type: image/png\r\n\r\n" with file_text with "\r\n--wfl-fixture-boundary--\r\n" + return http_request of site_spec and "/admin/media/upload" and "POST" and cookie_text and multipart_body and "multipart/form-data; boundary=wfl-fixture-boundary" +end action + +define action called http_copy_owned_data with parameters source_dir and target_dir: + // Only caller-created synthetic data paths may reach this backup helper. + call makedirs with target_dir + store entry_names as list_dir of source_dir + for each entry_name in entry_names: + store source_path as path_join of source_dir and entry_name + store target_path as path_join of target_dir and entry_name + check if is_dir of source_path: + call http_copy_owned_data with source_path and target_path + otherwise: + call copy_file with source_path and target_path + end check + end for +end action diff --git a/tests/integration/fixtures/legacy-extension.wfl b/tests/integration/fixtures/legacy-extension.wfl new file mode 100644 index 0000000..9246c80 --- /dev/null +++ b/tests/integration/fixtures/legacy-extension.wfl @@ -0,0 +1,26 @@ +// Copied by WFL into the disposable site's extension seam. +include from "render.wfl" + +define action called site_ext_boot with parameters db: + store created as execute db with "CREATE TABLE IF NOT EXISTS fixture_boots (id INTEGER PRIMARY KEY AUTOINCREMENT, marker TEXT NOT NULL)" + store inserted as execute db with "INSERT INTO fixture_boots (marker) VALUES ('owned extension boot')" + return yes +end action + +define action called site_ext_dispatch with parameters db and req and req_method and req_path and req_body and user: + check if req_method is equal to "GET" and (req_path is equal to "/" or req_path is equal to "/extension-status"): + store boot_rows as query db with "SELECT COUNT(*) AS total FROM fixture_boots" + store boot_row as boot_rows[0] + store note_rows as query db with "SELECT note FROM fixture_notes WHERE id = 5" + store note_row as note_rows[0] + create map ctx: + "site" is site_context of db and user + "boot_count" is boot_row["total"] + "retained_note" is note_row["note"] + end map + store body_html as render_public of "extension.html" and ctx + respond to req with body_html and content_type "text/html" + return yes + end check + return no +end action diff --git a/tests/integration/fixtures/legacy-site.wfl b/tests/integration/fixtures/legacy-site.wfl new file mode 100644 index 0000000..ce47087 --- /dev/null +++ b/tests/integration/fixtures/legacy-site.wfl @@ -0,0 +1,33 @@ +// Synthetic legacy schema, frozen from the original application at 4ec5c88. +// Deliberately omit sessions.csrf_token and all migration-history objects. +include from "http-helpers.wfl" + +define action called http_seed_legacy_site with parameters site_spec: + call makedirs with (path_dirname of site_spec["database"]) + open database at ("sqlite://" with site_spec["database"]) as legacy_db + try: + in transaction on legacy_db: + for each schema_sql in ["CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'author', created_at TEXT DEFAULT (datetime('now')))", "CREATE TABLE sessions (id TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL)", "CREATE TABLE posts (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT UNIQUE NOT NULL, title TEXT NOT NULL, body_markdown TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'draft', author_id INTEGER, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')))", "CREATE TABLE pages (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT UNIQUE NOT NULL, title TEXT NOT NULL, body_markdown TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'draft', author_id INTEGER, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')))", "CREATE TABLE settings (skey TEXT PRIMARY KEY, svalue TEXT NOT NULL DEFAULT '')", "CREATE TABLE media (id INTEGER PRIMARY KEY AUTOINCREMENT, filename TEXT UNIQUE NOT NULL, original_name TEXT NOT NULL DEFAULT '', content_type TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0, uploader_id INTEGER, created_at TEXT DEFAULT (datetime('now')))", "CREATE TABLE login_attempts (id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, attempted_at TEXT DEFAULT (datetime('now')))", "CREATE INDEX idx_login_attempts_ip_time ON login_attempts (ip, attempted_at)"]: + store created as execute legacy_db with schema_sql + end for + store password_hash as hash_password of "Synthetic-legacy-2026!" + store seeded_user as execute legacy_db with "INSERT INTO users (id, username, password_hash, role) VALUES (7, 'legacy-admin', ?, 'admin')" and parameters [password_hash] + store seeded_settings as execute legacy_db with "INSERT INTO settings (skey, svalue) VALUES ('installed', 'yes'), ('site_title', 'Legacy Extension Journal'), ('site_tagline', 'Retained legacy tagline'), ('posts_per_page', '3')" + store seeded_post as execute legacy_db with "INSERT INTO posts (id, slug, title, body_markdown, status, author_id) VALUES (12, 'heritage', 'Retained title', 'Retained **legacy body**', 'published', 7)" + store seeded_session as execute legacy_db with "INSERT INTO sessions (id, user_id, expires_at) VALUES ('synthetic-legacy-session', 7, '2099-01-01 00:00:00')" + store extension_table as execute legacy_db with "CREATE TABLE fixture_notes (id INTEGER PRIMARY KEY, note TEXT NOT NULL)" + store extension_row as execute legacy_db with "INSERT INTO fixture_notes (id, note) VALUES (5, 'Retained extension payload')" + end transaction + finally: + close database legacy_db + end try +end action + +define action called http_install_legacy_extension with parameters site_spec: + call copy_file with (path_join of http_fixture_root and "legacy-extension.wfl") and (path_join of site_spec["site"] and "app/site_ext.wfl") + store custom_theme_root as path_join of site_spec["site"] and "custom-themes/heritage/body" + call http_write with (path_join of custom_theme_root and "extension.html") and "{{ site.title }}
Custom extension theme | {{ site.title }} | {{ retained_note }} | Boots: {{ boot_count }}
" + call http_write with (path_join of custom_theme_root and "post.html") and "{{ post.title }}
Custom post theme | {{ site.title }} | {{ post.title }} | {{ post.author_name }} | {{ post.body_markdown | markdown }}
" + store site_config as http_base_config with "data_dir = runtime-data\nweb_server_port = " with site_spec["port"] with "\ntheme_root = custom-themes\ntheme = heritage\n" + call http_write with (path_join of site_spec["site"] and ".wflcfg") and site_config +end action diff --git a/tests/integration/fixtures/request.wfl b/tests/integration/fixtures/request.wfl new file mode 100644 index 0000000..cb5009e --- /dev/null +++ b/tests/integration/fixtures/request.wfl @@ -0,0 +1,15 @@ +// A native WFL HTTP request with its own five-second execution budget. +// The WFL driver owns this process and adds a six-second completion deadline. +open file at args[0] for reading as request_file +wait for store request_text as read content from request_file +close file request_file +store request_spec as parse_json of request_text +store request_url as request_spec["url"] +store request_method as request_spec["method"] +store request_body as request_spec["body"] +create map request_headers: + "Cookie" is request_spec["cookie"] + "Content-Type" is request_spec["content_type"] +end map +open url at request_url with method request_method and headers request_headers and body request_body and without following redirects and read response as request_reply +display stringify_json of request_reply diff --git a/tests/integration/legacy-upgrade.test.wfl b/tests/integration/legacy-upgrade.test.wfl new file mode 100644 index 0000000..7265058 --- /dev/null +++ b/tests/integration/legacy-upgrade.test.wfl @@ -0,0 +1,80 @@ +include from "fixtures/legacy-site.wfl" + + +define action called legacy_history_signature with parameters site_spec: + store history_rows as http_rows of site_spec and "SELECT position, migration_id, checksum, applied_at FROM _orm_migrations ORDER BY position" + store ordered_values as [] + for each history_row in history_rows: + push with ordered_values and [history_row["position"], history_row["migration_id"], history_row["checksum"], history_row["applied_at"]] + end for + return stringify_json of ordered_values +end action +define action called verify_legacy_http with parameters site_spec and expected_boots: + store locked_get as http_get of site_spec and "/install" and "" + call http_redirect with locked_get and "/" + store locked_post as http_post of site_spec and "/install" and "" and "site_title=Overwritten&username=intruder&password=Synthetic-pass-2026!" + call http_redirect with locked_post and "/" + for each route_path in ["/", "/extension-status"]: + store extension_reply as http_get of site_spec and route_path and "" + expect extension_reply["status"] to equal 200 + expect extension_reply["body"] to contain "Custom extension theme" + expect extension_reply["body"] to contain "Legacy Extension Journal" + expect extension_reply["body"] to contain "Retained extension payload" + expect extension_reply["body"] to contain ("Boots: " with expected_boots) + end for + store post_reply as http_get of site_spec and "/post/heritage" and "" + expect post_reply["status"] to equal 200 + expect post_reply["body"] to contain "Custom post theme" + expect post_reply["body"] to contain "Retained title" + expect post_reply["body"] to contain "Legacy Extension Journal" + expect post_reply["body"] to contain "legacy-admin" + expect post_reply["body"] to contain "legacy body" + store legacy_dashboard as http_get of site_spec and "/admin" and "sid=synthetic-legacy-session" + expect legacy_dashboard["status"] to equal 200 + store rejected_mutation as http_post of site_spec and "/admin/settings" and "sid=synthetic-legacy-session" and "site_title=Overwritten&csrf_token=" + expect rejected_mutation["status"] to equal 403 + store login_cookie as http_login of site_spec and "legacy-admin" and "Synthetic-legacy-2026!" + store login_dashboard as http_get of site_spec and "/admin" and login_cookie + expect login_dashboard["status"] to equal 200 + expect login_dashboard["body"] to contain "Legacy Extension Journal" + expect http_total of site_spec and "users" to equal 1 + expect http_total of site_spec and "posts" to equal 1 + expect http_total of site_spec and "fixture_notes" to equal 1 + expect http_total of site_spec and "fixture_boots" to equal expected_boots + expect http_total of site_spec and "_orm_migrations" to equal 2 +end action + +describe "Legacy HTTP upgrade with deployment extensions": + test "adoption and restart preserve installed users custom routes theme and extension state": + store site_spec as http_new_site of "configured" + try: + call http_seed_legacy_site with site_spec + call http_install_legacy_extension with site_spec + expect length of (http_rows of site_spec and "SELECT name FROM sqlite_master WHERE name = '_orm_migrations'") to equal 0 + expect length of (http_rows of site_spec and "SELECT name FROM pragma_table_info('sessions') WHERE name = 'csrf_token'") to equal 0 + store migrated_history as "" + store running_site as http_launch of site_spec + try: + call verify_legacy_http with site_spec and 1 + change migrated_history to legacy_history_signature of site_spec + store session_row as http_first_row of site_spec and "SELECT csrf_token FROM sessions WHERE id = 'synthetic-legacy-session'" + expect session_row["csrf_token"] to equal "" + expect running_site["startup"] to contain "Scriptorium theme: heritage" + finally: + call http_stop with running_site + end try + store restarted as http_launch of site_spec + try: + call verify_legacy_http with site_spec and 2 + expect (legacy_history_signature of site_spec) to equal migrated_history + store integrity as http_first_row of site_spec and "PRAGMA integrity_check" + expect integrity["integrity_check"] to equal "ok" + expect length of (http_rows of site_spec and "PRAGMA foreign_key_check") to equal 0 + finally: + call http_stop with restarted + end try + finally: + call http_destroy with site_spec + end try + end test +end describe diff --git a/tests/integration/media.test.wfl b/tests/integration/media.test.wfl new file mode 100644 index 0000000..ef4c865 --- /dev/null +++ b/tests/integration/media.test.wfl @@ -0,0 +1,112 @@ +include from "fixtures/http-helpers.wfl" + +describe "HTTP media storage and rejection boundaries": + test "configured and legacy data paths serve synthetic uploads and delete consistently": + for each config_mode in ["configured", "legacy"]: + store site_spec as http_new_site of config_mode + try: + store running_site as http_launch of site_spec + try: + store admin_cookie as http_install of site_spec + store admin_token as http_session_token of site_spec and admin_cookie + // The CMS validates extensions, not image decoding. These + // bytes intentionally test that implemented contract. + store uploaded as http_upload of site_spec and admin_cookie and admin_token and "../../unsafe.png" and "synthetic image payload" + call http_redirect with uploaded and "/admin/media" + store media_row as http_first_row of site_spec and "SELECT id, filename, original_name, content_type, size FROM media" + expect media_row["filename"] contains "/" to be no + expect media_row["filename"] contains "\\" to be no + expect media_row["filename"] contains ".." to be no + expect media_row["filename"] ends with ".png" to be yes + expect media_row["filename"] is not equal to "unsafe.png" to be yes + expect media_row["content_type"] to equal "image/png" + expect media_row["size"] to equal 23 + store upload_path as path_join of site_spec["uploads"] and media_row["filename"] + expect is_file of upload_path to be yes + store asset_route as "/assets/uploads/" with media_row["filename"] + store fetched as http_get of site_spec and asset_route and "" + expect fetched["status"] to equal 200 + expect fetched["body"] to equal "synthetic image payload" + store fetched_headers as fetched["headers"] + expect fetched_headers["content-type"] to contain "image/png" + store wrong_csrf as http_upload of site_spec and admin_cookie and "wrong" and "rejected.png" and "rejected" + expect wrong_csrf["status"] to equal 403 + store missing_csrf as http_upload of site_spec and admin_cookie and "" and "rejected.png" and "rejected" + expect missing_csrf["status"] to equal 403 + store bad_extension as http_upload of site_spec and admin_cookie and admin_token and "rejected.html" and "" + expect bad_extension["status"] to equal 200 + expect bad_extension["body"] to contain "Only png, jpg, jpeg, gif, and webp" + store empty_file as http_upload of site_spec and admin_cookie and admin_token and "empty.png" and "" + expect empty_file["status"] to equal 200 + expect empty_file["body"] to contain "That file was empty" + store malformed as http_request of site_spec and "/admin/media/upload" and "POST" and admin_cookie and "not multipart" and "multipart/form-data" + expect malformed["status"] to equal 200 + expect malformed["body"] to contain "file upload" + expect http_total of site_spec and "media" to equal 1 + expect length of (list_dir of site_spec["uploads"]) to equal 1 + store author_created as http_post of site_spec and "/admin/users" and admin_cookie and ("username=author&password=Author-pass-2026!&role=author&csrf_token=" with admin_token) + call http_redirect with author_created and "/admin/users" + store author_cookie as http_login of site_spec and "author" and "Author-pass-2026!" + store author_token as http_session_token of site_spec and author_cookie + store delete_route as "/admin/media/" with media_row["id"] with "/delete" + store denied_delete as http_post of site_spec and delete_route and author_cookie and ("csrf_token=" with author_token) + expect denied_delete["status"] to equal 403 + store get_delete as http_get of site_spec and delete_route and admin_cookie + expect get_delete["status"] to equal 405 + expect http_total of site_spec and "media" to equal 1 + expect is_file of upload_path to be yes + store deleted as http_post of site_spec and delete_route and admin_cookie and ("csrf_token=" with admin_token) + call http_redirect with deleted and "/admin/media" + expect http_total of site_spec and "media" to equal 0 + expect is_file of upload_path to be no + store absent_asset as http_get of site_spec and asset_route and "" + expect absent_asset["status"] to equal 404 + finally: + call http_stop with running_site + end try + finally: + call http_destroy with site_spec + end try + end for + end test + + test "request body ceiling rejects an oversized upload before persistence": + // Keep the shipped 10 MiB setting covered. This disposable site uses a + // smaller ceiling so its real oversized body fits the transport buffer: + // an eager 16 MiB HTTP/1 upload can receive a reset when rejected early. + open file at (path_join of http_source_root and ".wflcfg") for reading as defaults_file + wait for store defaults_text as read content from defaults_file + close file defaults_file + expect defaults_text to contain "web_server_max_body_size = 10485760" + store site_spec as http_new_site of "configured" + try: + store config_path as path_join of site_spec["site"] and ".wflcfg" + open file at config_path for reading as config_file + wait for store config_text as read content from config_file + close file config_file + expect config_text to contain "web_server_max_body_size = 10485760" + change config_text to replace "web_server_max_body_size = 10485760" with "web_server_max_body_size = 1024" in config_text + call http_write with config_path and config_text + store running_site as http_launch of site_spec + try: + store admin_cookie as http_install of site_spec + store admin_token as http_session_token of site_spec and admin_cookie + store large_text as "x" + count from 1 to 11: + change large_text to large_text with large_text + end count + expect length of large_text to equal 2048 + store too_large as http_upload of site_spec and admin_cookie and admin_token and "large.png" and large_text + expect too_large["status"] to equal 413 + expect http_total of site_spec and "media" to equal 0 + expect length of (list_dir of site_spec["uploads"]) to equal 0 + store after_rejection as http_get of site_spec and "/admin/media" and admin_cookie + expect after_rejection["status"] to equal 200 + finally: + call http_stop with running_site + end try + finally: + call http_destroy with site_spec + end try + end test +end describe diff --git a/tests/integration/migrations-recovery.test.wfl b/tests/integration/migrations-recovery.test.wfl new file mode 100644 index 0000000..912c357 --- /dev/null +++ b/tests/integration/migrations-recovery.test.wfl @@ -0,0 +1,143 @@ +include from "../fixtures/migration-registry.wfl" +include from "../../scripts/test_support.wfl" + +store migration_runtime as call current_executable +store migration_worker as path_join of current_directory and "tests/fixtures/migration-worker.wfl" + +define action called migration_wait_ready with parameters child_process and signal_path: + store ready_seen as no + count from 1 to 200: + check if file exists at signal_path: + change ready_seen to yes + break + end check + check if (process child_process is running) is equal to no: + wait for process child_process to complete with timeout 1 and read result as stopped_child + display stopped_child["output"] with stopped_child["error"] + call raise_error with "Migration fixture exited before readiness" + end check + wait for 25 milliseconds + end count + expect ready_seen to be yes + return yes +end action + +define action called migration_remove_signals with parameters database_path: + for each suffix_text in [".ready", ".second-ready", ".release"]: + store owned_path as database_path with suffix_text + check if file exists at owned_path: + delete file at owned_path + end check + end for +end action + +describe "Real process migration ownership and recovery": + test "a bounded locked migration fails cleanly and killing its owner rolls back schema data and events": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store initialized as orm_migrate of session and registry and "20260920000002" + store ready_path as database_path with ".ready" + store release_path as database_path with ".release" + wait for spawn command migration_runtime with arguments [migration_worker, database_path, "hold", ready_path, release_path] as holder_process + try: + call migration_wait_ready with holder_process and ready_path + store began_at as current time in milliseconds + store refused as no + try: + store blocked_plan as orm_migrate of session and registry and "latest" + when error: + change refused to yes + expect error_message contains "5 seconds" to be yes + end try + store elapsed_ms as (current time in milliseconds) minus began_at + expect refused to be yes + expect elapsed_ms is greater than or equal to 4000 to be yes + expect elapsed_ms is less than 8000 to be yes + finally: + close process holder_process + end try + expect (process holder_process is running) to be no + store recovered_status as orm_migration_status of session and registry + expect length of recovered_status.applied to equal 2 + expect orm_schema_exists of session and "abandoned_schema" to be no + expect length of (orm_sql_query of session and "SELECT * FROM items WHERE title='uncommitted'" and []) to equal 0 + expect length of (orm_sql_query of session and "SELECT * FROM _orm_migration_events WHERE migration_id='uncommitted'" and []) to equal 0 + store completed as orm_migrate of session and registry and "latest" + expect length of completed.applied to equal 3 + expect length of (orm_sql_query of session and "PRAGMA foreign_key_check" and []) to equal 0 + finally: + call migration_fixture_remove with session and database_path + call migration_remove_signals with database_path + end try + end test + + test "two real same-target runners converge without duplicate ledger or events": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store first_signal as database_path with ".ready" + store second_signal as database_path with ".second-ready" + wait for spawn command migration_runtime with arguments [migration_worker, database_path, "apply", first_signal, "latest"] as first_process + try: + wait for spawn command migration_runtime with arguments [migration_worker, database_path, "apply", second_signal, "latest"] as second_process + try: + wait for process first_process to complete with timeout 20 and read result as first_result + wait for process second_process to complete with timeout 20 and read result as second_result + expect first_result["exit_code"] to equal 0 + expect second_result["exit_code"] to equal 0 + finally: + close process second_process + end try + finally: + close process first_process + end try + store registry as migration_fixture_registry of "fixture index source\n" + store checked_plan as orm_migration_status of session and registry + expect length of checked_plan.applied to equal 3 + expect length of (orm_sql_query of session and "SELECT * FROM _orm_migration_events" and []) to equal 3 + finally: + call migration_fixture_remove with session and database_path + call migration_remove_signals with database_path + end try + end test + + test "a competing owner advancing beyond the requested target produces a conflict instead of false success": + store database_path as migration_fixture_path + store session as orm_open of database_path + try: + store registry as migration_fixture_registry of "fixture index source\n" + store initialized as orm_migrate of session and registry and "20260920000001" + store first_signal as database_path with ".ready" + store second_signal as database_path with ".second-ready" + store release_path as database_path with ".release" + wait for spawn command migration_runtime with arguments [migration_worker, database_path, "advance", first_signal, release_path] as owner_process + try: + call migration_wait_ready with owner_process and first_signal + wait for spawn command migration_runtime with arguments [migration_worker, database_path, "apply", second_signal, "20260920000002"] as waiting_process + try: + call migration_wait_ready with waiting_process and second_signal + call test_write_text with release_path and "commit" + wait for process owner_process to complete with timeout 10 and read result as owner_result + wait for process waiting_process to complete with timeout 10 and read result as waiting_result + expect owner_result["exit_code"] to equal 0 + expect waiting_result["exit_code"] is not equal to 0 to be yes + // Scheduling may expose the competing commit in the first + // consistent plan or in the later locked recheck. Both + // must refuse the stale target without claiming success. + expect ((waiting_result["error"] contains "migration concurrency") or (waiting_result["error"] contains "migration target")) to be yes + finally: + close process waiting_process + end try + finally: + close process owner_process + end try + store checked_plan as orm_migration_status of session and registry + expect length of checked_plan.applied to equal 3 + finally: + call migration_fixture_remove with session and database_path + call migration_remove_signals with database_path + end try + end test +end describe diff --git a/tests/integration/recovery.test.wfl b/tests/integration/recovery.test.wfl new file mode 100644 index 0000000..f5f6e85 --- /dev/null +++ b/tests/integration/recovery.test.wfl @@ -0,0 +1,73 @@ +include from "fixtures/http-helpers.wfl" + +describe "Synthetic site restart and database-upload recovery": + test "restart and a stopped-site backup restore users settings content and matching uploads": + store site_spec as http_new_site of "configured" + try: + store running_site as http_launch of site_spec + store asset_route as "" + try: + store admin_cookie as http_install of site_spec + store admin_token as http_session_token of site_spec and admin_cookie + store saved_post as http_post of site_spec and "/admin/posts" and admin_cookie and ("title=Recovered+post&slug=recovered&body_markdown=Retained+body&status=published&csrf_token=" with admin_token) + call http_redirect with saved_post and "/admin/posts" + store saved_settings as http_post of site_spec and "/admin/settings" and admin_cookie and ("site_title=Recovery+Journal&site_tagline=Matching+backup&posts_per_page=3&csrf_token=" with admin_token) + call http_redirect with saved_settings and "/admin/settings" + store saved_media as http_upload of site_spec and admin_cookie and admin_token and "recovery.png" and "matching backup bytes" + call http_redirect with saved_media and "/admin/media" + store media_row as http_first_row of site_spec and "SELECT filename FROM media" + change asset_route to "/assets/uploads/" with media_row["filename"] + finally: + call http_stop with running_site + end try + store data_root as path_join of site_spec["site"] and "runtime-data" + store backup_root as path_join of site_spec["root"] and "backup" + // Copy only after the owned server is reaped. Copy the entire + // synthetic data directory, including SQLite sidecars if present. + call http_copy_owned_data with data_root and backup_root + store restarted as http_launch of site_spec + try: + store locked as http_get of site_spec and "/install" and "" + call http_redirect with locked and "/" + store public_post as http_get of site_spec and "/post/recovered" and "" + expect public_post["status"] to equal 200 + expect public_post["body"] to contain "Retained body" + expect public_post["body"] to contain "Recovery Journal" + store asset_reply as http_get of site_spec and asset_route and "" + expect asset_reply["body"] to equal "matching backup bytes" + store logged_cookie as http_login of site_spec and "admin" and "Synthetic-pass-2026!" + store live_token as http_session_token of site_spec and logged_cookie + store changed as http_post of site_spec and "/admin/settings" and logged_cookie and ("site_title=After+backup&posts_per_page=9&csrf_token=" with live_token) + call http_redirect with changed and "/admin/settings" + finally: + call http_stop with restarted + end try + call remove_dir with data_root and yes + call http_copy_owned_data with backup_root and data_root + store restored as http_launch of site_spec + try: + store public_home as http_get of site_spec and "/" and "" + expect public_home["body"] to contain "Recovery Journal" + expect public_home["body"] contains "After backup" to be no + store post_reply as http_get of site_spec and "/post/recovered" and "" + expect post_reply["status"] to equal 200 + expect post_reply["body"] to contain "Retained body" + store media_reply as http_get of site_spec and asset_route and "" + expect media_reply["status"] to equal 200 + expect media_reply["body"] to equal "matching backup bytes" + store restored_cookie as http_login of site_spec and "admin" and "Synthetic-pass-2026!" + expect restored_cookie starts with "sid=" to be yes + expect http_total of site_spec and "users" to equal 1 + expect http_total of site_spec and "posts" to equal 1 + expect http_total of site_spec and "media" to equal 1 + store integrity as http_first_row of site_spec and "PRAGMA integrity_check" + expect integrity["integrity_check"] to equal "ok" + expect length of (http_rows of site_spec and "PRAGMA foreign_key_check") to equal 0 + finally: + call http_stop with restored + end try + finally: + call http_destroy with site_spec + end try + end test +end describe diff --git a/tests/integration/server-port.test.wfl b/tests/integration/server-port.test.wfl new file mode 100644 index 0000000..78b2639 --- /dev/null +++ b/tests/integration/server-port.test.wfl @@ -0,0 +1,45 @@ +include from "fixtures/http-helpers.wfl" + +define action called assert_port_site with parameters config_mode: + // Keep the former hardcoded port free too, so a regression can start and + // report its actual URL. Never stop another process to acquire a port. + store default_port as http_available_port of 8080 + store site_spec as http_new_site of config_mode + try: + store running_site as http_launch of site_spec + try: + store installer as http_get of site_spec and "/install" and "" + expect installer["status"] to equal 200 + store response_headers as installer["headers"] + expect response_headers["content-type"] to contain "text/html" + expect installer["body"] to contain "Set up your site" + expect installer["body"] to contain "Scriptorium" + expect installer["body"] to contain "action=\"/install\"" + expect installer["body"] to contain "name=\"csrf_token\"" + store server_process as running_site["process"] + wait for read output from process server_process as later_output + store actual_output as running_site["startup"] with later_output + expect actual_output to contain ("Scriptorium is running at " with site_spec["url"]) + expect actual_output to contain ("First run — open " with site_spec["url"] with "/install") + when error: + display running_site["startup"] + call raise_error with error_message + finally: + call http_stop with running_site + end try + finally: + call http_destroy with site_spec + end try +end action + +describe "Real HTTP listening-port compatibility": + test "configured non-default port serves installer and startup URLs": + call assert_port_site with "configured" + end test + test "missing port setting keeps port 8080": + call assert_port_site with "missing" + end test + test "missing site configuration keeps port 8080 on loopback": + call assert_port_site with "absent" + end test +end describe diff --git a/tests/integration/test_server_port.py b/tests/integration/test_server_port.py deleted file mode 100644 index 06468c9..0000000 --- a/tests/integration/test_server_port.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Exercise the configured listening port through a disposable Scriptorium site. - -Run sequentially with ``python -m unittest discover -s tests/integration -v``. -WFL_EXECUTABLE may select a runtime; otherwise wfl must be on PATH. The default -port cases require loopback port 8080 to be free and never stop other services. -""" - -import http.client -import os -from pathlib import Path -import shutil -import socket -import subprocess -import tempfile -import time -import unittest - - -SOURCE = Path(__file__).resolve().parents[2] -STARTUP_SECONDS = 20 -BASE_CONFIG = ( - "web_server_bind_address = 127.0.0.1\n" - "timeout_seconds = 60\n" - "logging_enabled = false\n" - "debug_report_enabled = false\n" -) - - -class ServerPortTests(unittest.TestCase): - def setUp(self): - requested = os.environ.get("WFL_EXECUTABLE", "wfl") - executable = shutil.which(requested) - self.assertIsNotNone(executable, f"WFL executable not found: {requested}") - self.executable = str(Path(executable).resolve()) - self.temporary = tempfile.TemporaryDirectory(prefix="scriptorium-port-test-") - self.addCleanup(self.temporary.cleanup) - self.root = Path(self.temporary.name) / "site" - self.root.mkdir() - # The runtime searches parent directories for configuration, whereas - # Scriptorium reads only its own .wflcfg. Keep the absent-file test on - # loopback too, regardless of the developer's global runtime settings. - (self.root.parent / ".wflcfg").write_text(BASE_CONFIG, encoding="utf-8") - runtime_files = [ - Path("main.wfl"), - Path("lib/scribe/src/scribe.wfl"), - Path("admin/templates/install.html"), - *(Path("app") / name for name in - ("util.wfl", "db.wfl", "auth.wfl", "render.wfl", "site_ext.wfl")), - ] - for relative in runtime_files: - source = SOURCE / relative - self.assertTrue(source.is_file(), f"Missing runtime source: {source}") - destination = self.root / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, destination) - # Never copy a checkout's database, uploads, configuration, or .git. - (self.root / "static").mkdir() - self.log_path = self.root.parent / "server.log" - self.log = self.log_path.open("wb") - self.addCleanup(self.log.close) - self.process = None - self.addCleanup(self.stop_server) - - def stop_server(self): - if self.process is not None and self.process.poll() is None: - self.process.terminate() - try: - self.process.wait(timeout=5) - except subprocess.TimeoutExpired: - self.process.kill() - self.process.wait(timeout=5) - - def server_log(self): - return self.log_path.read_text(encoding="utf-8", errors="replace") - - def available_port(self, requested=0): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as reservation: - if os.name == "nt": - reservation.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) - else: - # Permit a previous test's TIME_WAIT sockets, but not a listener. - reservation.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - reservation.bind(("127.0.0.1", requested)) - except OSError as exc: - self.fail( - f"Prerequisite: loopback port {requested} must be free; " - f"no existing service was stopped ({exc})" - ) - return reservation.getsockname()[1] - - def assert_installer(self, expected_port, config): - if config is not None: - (self.root / ".wflcfg").write_text( - BASE_CONFIG + "data_dir = runtime-data\n" + config, encoding="utf-8" - ) - # Also keep the old hardcoded port free when proving the configured - # case, so a regression can start and explain its actual bind in logs. - self.available_port(8080) - self.process = subprocess.Popen( - [self.executable, "main.wfl"], cwd=self.root, - stdin=subprocess.DEVNULL, stdout=self.log, stderr=subprocess.STDOUT, - ) - deadline = time.monotonic() + STARTUP_SECONDS - last_error = "server has not responded" - while time.monotonic() < deadline: - if self.process.poll() is not None: - self.fail( - f"Server exited with {self.process.returncode} before serving " - f"port {expected_port}:\n{self.server_log()}" - ) - connection = http.client.HTTPConnection("127.0.0.1", expected_port, timeout=1) - try: - connection.request("GET", "/install") - response = connection.getresponse() - body = response.read().decode("utf-8") - except (OSError, http.client.HTTPException) as exc: - last_error = str(exc) - else: - diagnostic = self.server_log() - self.assertEqual(response.status, 200, diagnostic + "\n" + body) - self.assertIn("text/html", response.getheader("Content-Type", "")) - self.assertIn("Set up your site", body) - self.assertIn('action="/install"', body) - self.assertIn('name="csrf_token"', body) - self.assertIn("Scriptorium", body) - self.assertIn( - f"Scriptorium is running at http://127.0.0.1:{expected_port}", diagnostic - ) - self.assertIn( - f"First run — open http://127.0.0.1:{expected_port}/install", diagnostic - ) - return - finally: - connection.close() - time.sleep(0.1) - self.fail( - f"No installer response on configured port {expected_port} within " - f"{STARTUP_SECONDS}s ({last_error}). Server output:\n{self.server_log()}" - ) - - def test_configured_port_serves_installer_and_updates_startup_urls(self): - selected = self.available_port() - self.assertNotEqual(selected, 8080, "OS ephemeral range must exclude default port 8080") - self.assert_installer(selected, f"web_server_port = {selected}\n") - - def test_missing_port_setting_defaults_to_8080(self): - self.assert_installer(8080, "# web_server_port intentionally omitted\n") - - def test_missing_config_file_defaults_to_8080(self): - self.assert_installer(8080, None) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/integration/throttle.test.wfl b/tests/integration/throttle.test.wfl new file mode 100644 index 0000000..c393abb --- /dev/null +++ b/tests/integration/throttle.test.wfl @@ -0,0 +1,43 @@ +include from "fixtures/http-helpers.wfl" + +describe "Persisted HTTP login throttling": + test "ten failed credentials block the next request and expired attempts recover": + store site_spec as http_new_site of "configured" + try: + store running_site as http_launch of site_spec + store login_token as "" + store login_cookie as "" + try: + store installed_cookie as http_install of site_spec + store login_form as http_get of site_spec and "/admin/login" and "" + change login_token to http_csrf of login_form + change login_cookie to http_cookie of login_form and "csrf" + count from 1 to 10: + store rejected as http_post of site_spec and "/admin/login" and login_cookie and ("username=nonexistent&password=wrong&csrf_token=" with login_token) + expect rejected["status"] to equal 200 + expect rejected["body"] to contain "Wrong username or password" + end count + expect http_total of site_spec and "login_attempts" to equal 10 + store throttled as http_post of site_spec and "/admin/login" and login_cookie and ("username=admin&password=Synthetic-pass-2026!&csrf_token=" with login_token) + expect throttled["status"] to equal 429 + expect http_total of site_spec and "sessions" to equal 1 + finally: + call http_stop with running_site + end try + store restarted as http_launch of site_spec + try: + store blocked_after_restart as http_post of site_spec and "/admin/login" and login_cookie and ("username=admin&password=Synthetic-pass-2026!&csrf_token=" with login_token) + expect blocked_after_restart["status"] to equal 429 + store old_attempts as http_execute of site_spec and "UPDATE login_attempts SET attempted_at = '2000-01-01 00:00:00'" + store recovered_cookie as http_login of site_spec and "admin" and "Synthetic-pass-2026!" + store dashboard as http_get of site_spec and "/admin" and recovered_cookie + expect dashboard["status"] to equal 200 + expect http_total of site_spec and "login_attempts" to equal 0 + finally: + call http_stop with restarted + end try + finally: + call http_destroy with site_spec + end try + end test +end describe diff --git a/tests/runtime/.wflcfg b/tests/runtime/.wflcfg new file mode 100644 index 0000000..2f0ae92 --- /dev/null +++ b/tests/runtime/.wflcfg @@ -0,0 +1,10 @@ +# Dedicated capability probes; application process execution remains disabled. +allow_shell_execution = true +# The probes launch current_executable by its exact absolute identity, including +# uninstalled validation candidates. Process permission is confined to tests. +shell_execution_mode = sanitized +timeout_seconds = 20 +execution_logging = false +debug_report_enabled = false +kill_on_shutdown = true +web_server_bind_address = 127.0.0.1 diff --git a/tests/runtime/fixtures/http-redirect.wfl b/tests/runtime/fixtures/http-redirect.wfl new file mode 100644 index 0000000..658fd75 --- /dev/null +++ b/tests/runtime/fixtures/http-redirect.wfl @@ -0,0 +1,15 @@ +// Loopback-only synthetic server. The parent owns and kills this process. +listen on port 41868 as probe_server +store readiness_path as args[0] +open file at readiness_path for writing as readiness_file +wait for write content "listening" into readiness_file +close file readiness_file +wait for request comes in on probe_server as incoming +create map redirect_headers: + "Location" is "/final" + "Set-Cookie" is "sid=synthetic-session; Path=/; HttpOnly" +end map +respond to incoming with "redirect body" and status 302 and headers redirect_headers +wait for request comes in on probe_server as final_incoming +respond to final_incoming with "final response" and status 200 +close server probe_server diff --git a/tests/runtime/fixtures/process-blocking.wfl b/tests/runtime/fixtures/process-blocking.wfl new file mode 100644 index 0000000..ff57f54 --- /dev/null +++ b/tests/runtime/fixtures/process-blocking.wfl @@ -0,0 +1,2 @@ +display "synthetic blocking child started" +wait for 15000 milliseconds diff --git a/tests/runtime/fixtures/process-child.wfl b/tests/runtime/fixtures/process-child.wfl new file mode 100644 index 0000000..accdebe --- /dev/null +++ b/tests/runtime/fixtures/process-child.wfl @@ -0,0 +1,2 @@ +display "synthetic child output" +display current_directory diff --git a/tests/runtime/fixtures/process-error.wfl b/tests/runtime/fixtures/process-error.wfl new file mode 100644 index 0000000..dd40510 --- /dev/null +++ b/tests/runtime/fixtures/process-error.wfl @@ -0,0 +1,3 @@ +// A controlled runtime failure gives the parent real stderr to preserve. +store numeric_result as 1 divided by 0 +display numeric_result diff --git a/tests/runtime/http-redirect-capability.test.wfl b/tests/runtime/http-redirect-capability.test.wfl new file mode 100644 index 0000000..5c1b229 --- /dev/null +++ b/tests/runtime/http-redirect-capability.test.wfl @@ -0,0 +1,52 @@ +// Capability gate: a CMS integration driver must inspect the response to its +// submitted request, including login redirects and the session cookie. +// Baseline Red is preserved in a81ecf9; loopback port 41868 must be free. +store runtime_path as call current_executable +describe "WFL HTTP contracts required by CMS integration": + test "login redirect status and session cookie remain observable": + // Refuse an occupied port; never send test requests to another service. + listen on port 41868 as port_reservation + close server port_reservation + store fixture_directory as path_join of script_directory and "fixtures" + store fixture_path as path_join of fixture_directory and "http-redirect.wfl" + store readiness_root as path_join of current_directory and "target/runtime-capability-results" + store readiness_id as generate_uuid + store readiness_directory as path_join of readiness_root and readiness_id + store readiness_path as path_join of readiness_directory and "http-ready.txt" + store observed_status as 0 + store observed_cookie as "" + try: + call makedirs with readiness_directory + wait for spawn command runtime_path with arguments [fixture_path, readiness_path] as child_process + try: + count from 1 to 100: + check if file exists at readiness_path: + break + end check + check if (process child_process is running) is no: + break + end check + wait for 100 milliseconds + end count + expect (is_file of readiness_path) to equal yes + expect (process child_process is running) to equal yes + open url at "http://127.0.0.1:41868/login" with method "POST" and body "synthetic=yes" and without following redirects and read response as probe_response + change observed_status to probe_response["status"] + store response_headers as probe_response["headers"] + check if response_headers contains "set-cookie": + change observed_cookie to response_headers["set-cookie"] + end check + finally: + close process child_process + end try + finally: + check if is_dir of readiness_directory: + call remove_dir with readiness_directory and yes + end check + end try + expect is_dir of readiness_directory to be no + display "Observed redirect response: status=" with observed_status with "; cookie=" with observed_cookie + expect observed_status to equal 302 + expect observed_cookie to contain "sid=synthetic-session" + end test +end describe diff --git a/tests/runtime/orm-native-baseline.test.wfl b/tests/runtime/orm-native-baseline.test.wfl new file mode 100644 index 0000000..5c4a8e6 --- /dev/null +++ b/tests/runtime/orm-native-baseline.test.wfl @@ -0,0 +1,114 @@ +// Passing characterization, including explicitly documented limitations. +store checkout_root as path_dirname of (path_dirname of script_directory) +store probe_dir as path_join of checkout_root and "target" and "runtime-capability-results" +create directory at probe_dir +store probe_file as path_join of probe_dir and "orm-native-baseline.db" +store db_url as "sqlite://" with probe_file + +define action called insert_then_return_failure with parameters conn: + in transaction on conn: + store ins as execute conn with "INSERT INTO commit_return (label) VALUES ('retained')" + return no + end transaction +end action + +create container ProbeRecord: + property title: Text + action rename needs new_title: Text: + change title to new_title + end +end + +open database at db_url as conn +try: + describe "ORM runtime capabilities": + test "typed containers mutate properties through actions": + create new ProbeRecord as a_record: + title is "before" + end + a_record.rename("after") + expect a_record.title to equal "after" + end test + test "map presence distinguishes missing and null; JSON null has same nothing comparison": + create map a_record: + "nullable" is nothing + "empty" is "" + end map + expect a_record contains "nullable" to be yes + expect a_record contains "absent" to be no + expect a_record["nullable"] to equal nothing + expect a_record["empty"] to equal "" + store json_null as parse_json of "null" + expect isnothing of json_null to be yes + expect json_null to equal nothing + display "nothing literal type: " with (typeof of nothing) + display "JSON null type: " with (typeof of json_null) + end test + test "parameters preserve injection payloads and null; sqlite schema is inspectable": + + store dropped as execute conn with "DROP TABLE IF EXISTS parameter_case" + store made as execute conn with "CREATE TABLE parameter_case (id INTEGER PRIMARY KEY, title TEXT UNIQUE, note TEXT)" + store malicious_text as "x'; DROP TABLE parameter_case; --" + store ins as execute conn with "INSERT INTO parameter_case (title, note) VALUES (?, ?)" and parameters [malicious_text and nothing] + expect ins["affected_rows"] to equal 1 + expect ins["last_insert_id"] to equal 1 + store rows as query conn with "SELECT id, title, note FROM parameter_case WHERE title = ?" and parameters [malicious_text] + store a_row as rows[0] + expect a_row["note"] to equal nothing + expect a_row["title"] to equal malicious_text + store cols as query conn with "SELECT name, type, pk FROM pragma_table_info(?) ORDER BY cid" and parameters ["parameter_case"] + expect length of cols to equal 3 + store schema_rows as query conn with "SELECT sql FROM sqlite_schema WHERE type = ? AND name = ?" and parameters ["table" and "parameter_case"] + expect length of schema_rows to equal 1 + + end test + test "failed DDL and ledger changes roll back together and original diagnostics propagate": + + store dropped as execute conn with "DROP TABLE IF EXISTS atomic_ledger" + store dropped2 as execute conn with "DROP TABLE IF EXISTS atomic_new_table" + store made as execute conn with "CREATE TABLE atomic_ledger (version TEXT UNIQUE)" + store caught_message as "" + try: + in transaction on conn: + store made2 as execute conn with "CREATE TABLE atomic_new_table (id INTEGER)" + store ins as execute conn with "INSERT INTO atomic_ledger (version) VALUES ('v1')" + store broken as execute conn with "INSERT INTO missing_required_table (id) VALUES (1)" + end transaction + when error: + change caught_message to error_message + end try + expect caught_message contains "missing_required_table" to be yes + store ledger_rows as query conn with "SELECT version FROM atomic_ledger" + expect length of ledger_rows to equal 0 + store schema_rows as query conn with "SELECT name FROM sqlite_schema WHERE name = 'atomic_new_table'" + expect length of schema_rows to equal 0 + + end test + test "returning a validation failure is a successful transaction and commits": + + store dropped as execute conn with "DROP TABLE IF EXISTS commit_return" + store made as execute conn with "CREATE TABLE commit_return (label TEXT)" + store returned as insert_then_return_failure of conn + expect returned to be no + store rows as query conn with "SELECT label FROM commit_return" + expect length of rows to equal 1 + + end test + test "integer identifiers above 2 to 53 lose precision on reads": + + store rows as query conn with "SELECT 9007199254740993 AS precise_id, CAST(9007199254740993 AS TEXT) AS text_id" + store a_row as rows[0] + expect a_row["text_id"] to equal "9007199254740993" + display "integer mapped to WFL: " with a_row["precise_id"] + store roundtrip as query conn with "SELECT CAST(? AS TEXT) AS bound_id" and parameters [a_row["precise_id"]] + store bound_row as roundtrip[0] + expect bound_row["bound_id"] to equal "9007199254740992" + + end test + end describe + + +finally: + close database conn + delete file at probe_file +end try diff --git a/tests/runtime/process-capabilities.test.wfl b/tests/runtime/process-capabilities.test.wfl new file mode 100644 index 0000000..0491187 --- /dev/null +++ b/tests/runtime/process-capabilities.test.wfl @@ -0,0 +1,54 @@ +// Original behavioral Red lives in a81ecf9. Use the reviewed native facilities +// while preserving working-directory, complete diagnostics and cleanup claims. +store runtime_path as call current_executable + +describe "WFL process contracts required by the complete suite": + test "direct children use an isolated fixture working directory": + store fixture_directory as path_join of script_directory and "fixtures" + store fixture_path as path_join of fixture_directory and "process-child.wfl" + wait for execute command runtime_path with arguments [fixture_path] in directory fixture_directory as child_result + expect child_result["exit_code"] to equal 0 + expect child_result["output"] to contain fixture_directory + end test + + test "completion returns final captured output with its exit status": + store fixture_path as path_join of script_directory and "fixtures/process-child.wfl" + wait for spawn command runtime_path with arguments [fixture_path] as child_process + try: + wait for process child_process to complete with timeout 10 and read result as outcome + expect outcome["exit_code"] to equal 0 + expect outcome["output"] to contain "synthetic child output" + finally: + close process child_process + end try + end test + + test "background process diagnostics preserve stderr": + store fixture_path as path_join of script_directory and "fixtures/process-error.wfl" + wait for spawn command runtime_path with arguments [fixture_path] as child_process + try: + wait for process child_process to complete with timeout 10 and read result as outcome + expect outcome["exit_code"] to equal 1 + expect outcome["error"] to contain "Division by zero" + finally: + close process child_process + end try + end test + + test "finally terminates a directly owned child after a controlled error": + store fixture_path as path_join of script_directory and "fixtures/process-blocking.wfl" + wait for spawn command runtime_path with arguments [fixture_path] as child_process + store observed_error as no + try: + wait for 300 milliseconds + expect (process child_process is running) to equal yes + call raise_error with "controlled process owner failure" + when error: + change observed_error to yes + finally: + close process child_process + end try + expect observed_error to equal yes + expect (process child_process is running) to equal no + end test +end describe diff --git a/tests/runtime/rebuild-foreign-keys-capability.test.wfl b/tests/runtime/rebuild-foreign-keys-capability.test.wfl new file mode 100644 index 0000000..24689a8 --- /dev/null +++ b/tests/runtime/rebuild-foreign-keys-capability.test.wfl @@ -0,0 +1,39 @@ +// Baseline Red is preserved in a81ecf9. Explicit schema mode disables foreign +// keys on the owned connection before BEGIN and checks integrity before commit. +store checkout_root as path_dirname of (path_dirname of script_directory) +store probe_dir as path_join of checkout_root and "target" and "runtime-capability-results" +create directory at probe_dir +store probe_file as path_join of probe_dir and "rebuild-foreign-keys.db" +store probe_url as "sqlite://" with probe_file + +describe "Required table rebuild preservation capability": + test "rebuilding a managed parent preserves extension-owned referencing rows": + open database at probe_url as conn + try: + store dropped_child as execute conn with "DROP TABLE IF EXISTS extension_children" + store dropped_parent as execute conn with "DROP TABLE IF EXISTS managed_parent" + store dropped_replacement as execute conn with "DROP TABLE IF EXISTS replacement" + store made_parent as execute conn with "CREATE TABLE managed_parent (id INTEGER PRIMARY KEY, label TEXT)" + store made_child as execute conn with "CREATE TABLE extension_children (id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES managed_parent(id) ON DELETE CASCADE)" + store seeded_parent as execute conn with "INSERT INTO managed_parent VALUES (1, 'preserve')" + store seeded_child as execute conn with "INSERT INTO extension_children VALUES (1, 1)" + in transaction on conn for schema changes: + store fk_rows as query conn with "PRAGMA foreign_keys" + store fk_row as fk_rows[0] + display "foreign_keys inside transaction: " with fk_row["foreign_keys"] + store make_new as execute conn with "CREATE TABLE replacement (id INTEGER PRIMARY KEY, label TEXT, added TEXT)" + store copy_rows as execute conn with "INSERT INTO replacement (id, label) SELECT id, label FROM managed_parent" + store drop_old as execute conn with "DROP TABLE managed_parent" + store rename_new as execute conn with "ALTER TABLE replacement RENAME TO managed_parent" + end transaction + store children as query conn with "SELECT id, parent_id FROM extension_children" + expect length of children to equal 1 + store child_row as children[0] + expect child_row["id"] to equal 1 + expect child_row["parent_id"] to equal 1 + finally: + close database conn + delete file at probe_file + end try + end test +end describe diff --git a/tests/runtime/transaction-validation-capability.test.wfl b/tests/runtime/transaction-validation-capability.test.wfl new file mode 100644 index 0000000..2667a3f --- /dev/null +++ b/tests/runtime/transaction-validation-capability.test.wfl @@ -0,0 +1,37 @@ +// Baseline Red is preserved in a81ecf9: false returns committed a partial write. +// The deliberate error primitive now unwinds through the same native scope. +store checkout_root as path_dirname of (path_dirname of script_directory) +store probe_dir as path_join of checkout_root and "target" and "runtime-capability-results" +create directory at probe_dir +store probe_file as path_join of probe_dir and "transaction-validation.db" +store probe_url as "sqlite://" with probe_file + +define action called save_then_reject with parameters conn: + in transaction on conn: + store inserted as execute conn with "INSERT INTO validation_probe (label) VALUES (?)" and parameters ["must roll back"] + call raise_error with "controlled validation rejection" + end transaction +end action + +describe "Required validation failure rollback capability": + test "a failed operation must not leave an earlier write committed": + open database at probe_url as conn + try: + store dropped as execute conn with "DROP TABLE IF EXISTS validation_probe" + store created as execute conn with "CREATE TABLE validation_probe (label TEXT NOT NULL)" + store rejected as no + try: + call save_then_reject with conn + when error: + change rejected to yes + expect error_message contains "controlled validation rejection" to be yes + end try + expect rejected to be yes + store rows as query conn with "SELECT label FROM validation_probe" + expect length of rows to equal 0 + finally: + close database conn + delete file at probe_file + end try + end test +end describe diff --git a/tests/tooling/.wflcfg b/tests/tooling/.wflcfg new file mode 100644 index 0000000..767a576 --- /dev/null +++ b/tests/tooling/.wflcfg @@ -0,0 +1,7 @@ +timeout_seconds = 180 +execution_logging = false +debug_report_enabled = false +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true +max_buffer_size_bytes = 8388608 diff --git a/tests/tooling/fixtures/delayed-write.wfl b/tests/tooling/fixtures/delayed-write.wfl new file mode 100644 index 0000000..b3e08e9 --- /dev/null +++ b/tests/tooling/fixtures/delayed-write.wfl @@ -0,0 +1,9 @@ +store ready_path as path_join of args[0] and "descendant-ready.txt" +open file at ready_path for writing as ready_handle +wait for write content "ready" into ready_handle +close file ready_handle +wait for 3 seconds +store late_path as path_join of args[0] and "descendant-late.txt" +open file at late_path for writing as late_handle +wait for write content "must not survive runner timeout" into late_handle +close file late_handle diff --git a/tests/tooling/fixtures/runner-error.wfl b/tests/tooling/fixtures/runner-error.wfl new file mode 100644 index 0000000..45b0101 --- /dev/null +++ b/tests/tooling/fixtures/runner-error.wfl @@ -0,0 +1,3 @@ +display "intentional runtime stdout" +store denominator as 0 +display 1 divided by denominator diff --git a/tests/tooling/fixtures/runner-fail.wfl b/tests/tooling/fixtures/runner-fail.wfl new file mode 100644 index 0000000..4f9b7fe --- /dev/null +++ b/tests/tooling/fixtures/runner-fail.wfl @@ -0,0 +1,6 @@ +display "intentional fixture stdout" +describe "Synthetic runner failure": + test "intentional fixture failure": + expect "deliberate failure" to equal "success" + end test +end describe diff --git a/tests/tooling/fixtures/runner-pass.wfl b/tests/tooling/fixtures/runner-pass.wfl new file mode 100644 index 0000000..366f9aa --- /dev/null +++ b/tests/tooling/fixtures/runner-pass.wfl @@ -0,0 +1,10 @@ +store build_exists as is_dir of "build" +store stale_build as path_exists of "build/existing.txt" +store copied_git as path_exists of ".git" +store copied_cache as path_exists of "__pycache__" +display "RUN {\"suite\":" with (stringify_json of script_path) with ",\"cwd\":" with (stringify_json of current_directory) with ",\"build_exists\":" with (stringify_json of build_exists) with ",\"stale_build\":" with (stringify_json of stale_build) with ",\"copied_git\":" with (stringify_json of copied_git) with ",\"copied_cache\":" with (stringify_json of copied_cache) with "}" +describe "Synthetic runner success": + test "real WFL assertion passes": + expect yes to equal yes + end test +end describe diff --git a/tests/tooling/fixtures/runner-timeout.wfl b/tests/tooling/fixtures/runner-timeout.wfl new file mode 100644 index 0000000..6a144d0 --- /dev/null +++ b/tests/tooling/fixtures/runner-timeout.wfl @@ -0,0 +1,5 @@ +store runtime_path as call current_executable +display "diagnostic before runner timeout" +store child_path as path_join of current_directory and "helpers/delayed-write.wfl" +wait for spawn command runtime_path with arguments [child_path, current_directory] as owned_child +wait for 10 seconds diff --git a/tests/tooling/hygiene.test.wfl b/tests/tooling/hygiene.test.wfl new file mode 100644 index 0000000..e6d3739 --- /dev/null +++ b/tests/tooling/hygiene.test.wfl @@ -0,0 +1,272 @@ +// All fixtures and assertions are WFL. Python runs only the existing checker. +include from "../../scripts/test_support.wfl" + +store repo_root as path_dirname of (path_dirname of script_directory) +store checker_path as path_join of repo_root and "scripts/check_repo_hygiene.py" +store profile_text as test_read_text of (path_join of repo_root and ".repo-hygiene.toml") +store profile_value as parse_toml of profile_text +store python_program as "python" +// Optional args[1] selects the utility interpreter; args[0] is the WFL runtime. +check if (length of args) is greater than 1: + change python_program to args[1] +end check +store artifact_root as path_join of repo_root and "target/test-artifacts/hygiene" and (generate_uuid) +store fixture_root as "" + +define action called hygiene_git with parameters git_arguments: + store invocation as ["-C", fixture_root] + for each argument_value in git_arguments: + push with invocation and argument_value + end for + store outcome as test_execute of "git" and invocation and fixture_root and 10 + check if outcome["exit_code"] is not equal to 0: + display outcome["error"] + end check + expect outcome["exit_code"] to equal 0 + return trim of outcome["output"] +end action + +define action called hygiene_write with parameters relative_path and text_value: + call test_write_text with (path_join of fixture_root and relative_path) and text_value +end action + +define action called hygiene_binary with parameters relative_path and byte_values: + store destination as path_join of fixture_root and relative_path + call makedirs with (path_dirname of destination) + open file at destination for writing binary as output_file + try: + write binary byte_values into output_file + finally: + close file output_file + end try +end action + +define action called hygiene_result with parameters expected_code and fragment: + store outcome as test_execute of python_program and [checker_path, "--root", fixture_root] and fixture_root and 15 + store diagnostic_text as outcome["output"] with outcome["error"] + display diagnostic_text + expect outcome["exit_code"] to equal expected_code + expect diagnostic_text to contain fragment + return diagnostic_text +end action + +define action called hygiene_fixture: + change fixture_root to path_join of artifact_root and (generate_uuid) + call makedirs with fixture_root + call hygiene_git with ["init", "--quiet"] + for each required_path in profile_value["required"]["files"]: + call hygiene_write with required_path and ("Fixture file." with newline) + end for + call hygiene_write with ".repo-hygiene.toml" and profile_text + call hygiene_write with ".gitignore" and "*.db\n*.private\n.env\n__pycache__/\n" + call hygiene_write with "README.md" and "[Architecture](docs/ARCHITECTURE.md)\n" + call hygiene_write with "static/uploads/.gitkeep" and "" + call hygiene_write with "app/source.wfl" and "display \"fixture\"\n" + call hygiene_git with ["add", "."] + call hygiene_git with ["update-index", "--add", "--cacheinfo", "160000", "1111111111111111111111111111111111111111", "lib/scribe"] +end action + +describe "Repository hygiene regression requirements": + teardown: + check if is_dir of artifact_root: + call remove_dir with artifact_root and yes + end check + end teardown + + test "clean checkout and uninitialized submodule pass": + call hygiene_fixture + call hygiene_result with 0 and "Repository hygiene passed" + end test + + test "new untracked root file is checked before staging": + call hygiene_fixture + call hygiene_write with "scratch.md" and "Unapproved root note\n" + call hygiene_result with 1 and "HYGIENE: root: scratch.md" + end test + + test "new untracked root directory is rejected": + call hygiene_fixture + call hygiene_write with "notes/design.md" and "Unapproved root directory\n" + call hygiene_result with 1 and "HYGIENE: root: notes/design.md" + end test + + test "legitimate docs and binary source assets pass": + call hygiene_fixture + call hygiene_write with "docs/new-guide.md" and "Maintained guide\n" + call hygiene_binary with "static/fonts/new.woff2" and [119, 79, 70, 50, 0, 102, 105, 120, 116, 117, 114, 101] + call hygiene_binary with "docs/screenshots/new.png" and [137, 80, 78, 71, 13, 10, 26, 10, 0, 102, 105, 120, 116, 117, 114, 101] + call hygiene_result with 0 and "Repository hygiene passed" + end test + + test "ignored runtime data remains untouched": + call hygiene_fixture + call hygiene_write with "scriptorium.db" and "private runtime fixture" + call hygiene_write with ".env" and "TOKEN=fixture\n" + call hygiene_result with 0 and "Repository hygiene passed" + expect (test_read_text of (path_join of fixture_root and "scriptorium.db")) to equal "private runtime fixture" + end test + + test "force tracked ignored database is rejected": + call hygiene_fixture + call hygiene_write with "app/fixture.db" and "runtime fixture" + call hygiene_git with ["add", "--force", "app/fixture.db"] + call hygiene_result with 1 and "HYGIENE: artifact: app/fixture.db" + end test + + test "repository ignores all SQLite database and sidecar forms": + call hygiene_fixture + call hygiene_write with ".gitignore" and (test_read_text of (path_join of repo_root and ".gitignore")) + for each extension_name in ["db", "sqlite", "sqlite3"]: + for each suffix_value in ["", "-journal", "-wal", "-shm"]: + call hygiene_write with ("site." with extension_name with suffix_value) and "local runtime fixture" + end for + end for + call hygiene_result with 0 and "Repository hygiene passed" + end test + + test "case variants and nested cache are rejected": + call hygiene_fixture + call hygiene_write with ".gitignore" and "# No ignores in this test.\n" + for each candidate_path in ["app/site.SQLITE3-WAL", "docs/server.LOG", "scripts/__PyCache__/bad.txt"]: + call hygiene_write with candidate_path and "fixture\n" + call hygiene_result with 1 and ("HYGIENE: artifact: " with candidate_path) + call remove_file with (path_join of fixture_root and candidate_path) + end for + end test + + test "secret names are rejected at any depth": + call hygiene_fixture + for each candidate_path in ["app/.env.production", "docs/id_rsa", "app/secret.KEY"]: + call hygiene_write with candidate_path and "fixture\n" + call hygiene_result with 1 and ("HYGIENE: artifact: " with candidate_path) + call remove_file with (path_join of fixture_root and candidate_path) + end for + end test + + test "uploads cannot use the source image exception": + call hygiene_fixture + call hygiene_binary with "static/uploads/photo.png" and [137, 80, 78, 71, 0, 102, 105, 120, 116, 117, 114, 101] + call hygiene_result with 1 and "HYGIENE: runtime-state: static/uploads/photo.png" + end test + + test "only empty declared upload placeholder is allowed": + call hygiene_fixture + call hygiene_write with "static/uploads/.gitkeep" and "runtime bytes\n" + call hygiene_result with 1 and "HYGIENE: runtime-state: static/uploads/.gitkeep" + end test + + test "removed and empty required policies both fail": + call hygiene_fixture + call remove_file with (path_join of fixture_root and "GOVERNANCE.md") + call hygiene_result with 1 and "HYGIENE: required: GOVERNANCE.md" + call hygiene_write with "GOVERNANCE.md" and "" + call hygiene_result with 1 and "required file is empty" + end test + + test "ignored required policy cannot pass by existing locally": + call hygiene_fixture + call hygiene_git with ["rm", "--cached", "--force", "GOVERNANCE.md"] + call hygiene_write with ".gitignore" and "GOVERNANCE.md\n" + call hygiene_result with 1 and "HYGIENE: required: GOVERNANCE.md" + end test + + test "unstaged worktree link changes are checked": + call hygiene_fixture + call hygiene_write with "README.md" and "[Missing](docs/does-not-exist.md)\n" + call hygiene_result with 1 and "local link is missing from candidate tree" + end test + + test "ignored local file cannot hide a broken clone link": + call hygiene_fixture + call hygiene_write with "docs/local.private" and "private fixture\n" + call hygiene_write with "README.md" and "[Local](docs/local.private)\n" + call hygiene_result with 1 and "local link is missing from candidate tree" + end test + + test "new candidates directories and encoded paths are accepted": + call hygiene_fixture + call hygiene_write with "docs/new file.md" and "New guide\n" + call hygiene_write with "README.md" and "[Guide](docs/new%20file.md#title)\n[Docs](docs/)\n[Spaced]()\n[Ref][guide]\n[guide]: docs/new%20file.md \"A guide\"\n" + call hygiene_result with 0 and "Repository hygiene passed" + end test + + test "reference and image link destinations are both checked": + call hygiene_fixture + call hygiene_write with "README.md" and "![Image](docs/missing.png)\n[ref]: docs/missing.md\n" + store diagnostic_text as hygiene_result of 1 and "docs/missing.png" + expect diagnostic_text to contain "docs/missing.md" + end test + + test "anchors external URLs fences and submodule links are skipped": + call hygiene_fixture + call hygiene_write with "README.md" and "[Anchor](#unknown)\n[Web](https://example.invalid/nope)\n[Mail](mailto:test@example.invalid)\n[Scribe](lib/scribe/uninitialized.md)\n```md\n[Example](missing.md)\n```\n~~~md\n[Example](missing-too.md)\n~~~\n" + call hygiene_result with 0 and "Repository hygiene passed" + end test + + test "links cannot escape the checkout": + call hygiene_fixture + call hygiene_write with "README.md" and "[Outside](../outside.md)\n" + call hygiene_result with 1 and "local link leaves checkout" + end test + + test "submodule contents are not recursively inspected": + call hygiene_fixture + call hygiene_write with "lib/scribe/private.db" and "upstream runtime fixture\n" + call hygiene_write with "lib/scribe/.env" and "upstream fixture\n" + call hygiene_result with 0 and "Repository hygiene passed" + end test + + test "Scribe cannot be replaced with copied files": + call hygiene_fixture + call hygiene_git with ["update-index", "--force-remove", "lib/scribe"] + call hygiene_write with "lib/scribe/src/scribe.wfl" and "copied upstream fixture\n" + call hygiene_result with 1 and "required Git gitlink is missing" + end test + + test "another gitlink cannot bypass artifact inspection": + call hygiene_fixture + call hygiene_git with ["update-index", "--add", "--cacheinfo", "160000", "2222222222222222222222222222222222222222", "lib/other"] + call hygiene_result with 1 and "HYGIENE: submodule: lib/other: unapproved gitlink" + end test + + test "indexed symlink is rejected even as physical text": + call hygiene_fixture + call hygiene_write with "app/linked.wfl" and "../README.md" + store blob_identity as hygiene_git of ["hash-object", "-w", (path_join of fixture_root and "app/linked.wfl")] + call hygiene_git with ["update-index", "--add", "--cacheinfo", "120000", blob_identity, "app/linked.wfl"] + call hygiene_result with 1 and "HYGIENE: file-type: app/linked.wfl" + end test + + test "malformed profile fails closed": + call hygiene_fixture + call hygiene_write with ".repo-hygiene.toml" and "schema = [\n" + call hygiene_result with 2 and "HYGIENE-ERROR" + end test + + test "missing profile fails closed": + call hygiene_fixture + call remove_file with (path_join of fixture_root and ".repo-hygiene.toml") + call hygiene_result with 2 and "cannot read .repo-hygiene.toml" + end test + + test "unknown profile key fails closed": + call hygiene_fixture + call hygiene_write with ".repo-hygiene.toml" and (replace "allowed-dirs =" with "allowed-directories =" in profile_text) + call hygiene_result with 2 and "missing or unknown keys" + end test + + test "profile paths cannot escape checkout": + call hygiene_fixture + call hygiene_write with ".repo-hygiene.toml" and (replace "\"docs/ARCHITECTURE.md\"" with "\"../ARCHITECTURE.md\"" in profile_text) + call hygiene_result with 2 and "non-relative path" + end test + + test "Git failure fails closed": + call hygiene_fixture + call move_file with (path_join of fixture_root and ".git") and (path_join of fixture_root and "fixture-index-away") + // Artifacts live below this checkout. Stop Git's parent search so the + // missing fixture repository cannot accidentally resolve the real one. + call hygiene_write with ".git" and "gitdir: ./missing-fixture-index\n" + call hygiene_result with 2 and "HYGIENE-ERROR: git" + end test +end describe diff --git a/tests/tooling/migrations.test.wfl b/tests/tooling/migrations.test.wfl new file mode 100644 index 0000000..edd83a1 --- /dev/null +++ b/tests/tooling/migrations.test.wfl @@ -0,0 +1,191 @@ +include from "../../scripts/test_support.wfl" + +store migration_repo_root as path_dirname of (path_dirname of script_directory) +store migration_runtime as call current_executable + +define action called migration_cli_fixture: + store run_id as generate_uuid + store site_root as path_join of migration_repo_root and "target/test-artifacts/migration-cli" and run_id + try: + call makedirs with (path_join of site_root and "scripts") + call copy_file with (path_join of migration_repo_root and "scripts/migrate.wfl") and (path_join of site_root and "scripts/migrate.wfl") + call test_copy_tree with (path_join of migration_repo_root and "lib/orm") and (path_join of site_root and "lib/orm") + call test_copy_tree with (path_join of migration_repo_root and "app/migrations") and (path_join of site_root and "app/migrations") + call copy_file with (path_join of migration_repo_root and "app/migrations.wfl") and (path_join of site_root and "app/migrations.wfl") + call copy_file with (path_join of migration_repo_root and "app/util.wfl") and (path_join of site_root and "app/util.wfl") + call test_write_text with (path_join of site_root and ".wflcfg") and "execution_logging = false\nlogging_enabled = false\ndebug_report_enabled = false\n" + return site_root + when error: + check if is_dir of site_root: + call remove_dir with site_root and yes + end check + call raise_error with error_message + end try +end action + +define action called migration_cli_run with parameters site_root and argument_values: + store child_args as ["scripts/migrate.wfl"] + for each argument_value in argument_values: + push with child_args and argument_value + end for + return test_execute of migration_runtime and child_args and site_root and 30 +end action + +describe "Migration CLI target selection and lifecycle": + test "absent status and plan create no database or data directory with default or configured paths": + store site_root as migration_cli_fixture + try: + delete file at (path_join of site_root and ".wflcfg") + store default_status as migration_cli_run of site_root and ["status"] + expect default_status["exit_code"] to equal 0 + expect default_status["output"] contains "Database absent" to be yes + expect file exists at (path_join of site_root and "scriptorium.db") to be no + call test_write_text with (path_join of site_root and ".wflcfg") and "execution_logging = false\nlogging_enabled = false\ndebug_report_enabled = false\ndata_dir = configured-data\ndata_dir = ignored-second-value\n" + store configured_plan as migration_cli_run of site_root and ["plan"] + expect configured_plan["exit_code"] to equal 0 + expect configured_plan["output"] contains "configured-data" to be yes + expect configured_plan["output"] contains "Planned fresh apply: 2" to be yes + expect is_dir of (path_join of site_root and "configured-data") to be no + store override_plan as migration_cli_run of site_root and ["plan", "--database", "override/site.db", "--target", "20200101000000"] + expect override_plan["exit_code"] to equal 0 + expect override_plan["output"] contains "Planned fresh apply: 1" to be yes + expect is_dir of (path_join of site_root and "override") to be no + finally: + call remove_dir with site_root and yes + end try + end test + + test "configured targeted apply repeat refusal and changed-source errors have real process statuses": + store site_root as migration_cli_fixture + try: + call test_write_text with (path_join of site_root and ".wflcfg") and "execution_logging = false\nlogging_enabled = false\ndebug_report_enabled = false\ndata_dir = configured-data\n" + store first_apply as migration_cli_run of site_root and ["up", "--target", "20200101000000"] + expect first_apply["exit_code"] to equal 0 + expect first_apply["output"] contains "Applied migrations: 1" to be yes + store next_apply as migration_cli_run of site_root and ["up"] + expect next_apply["exit_code"] to equal 0 + expect next_apply["output"] contains "Applied migrations: 2" to be yes + store repeated as migration_cli_run of site_root and ["up"] + expect repeated["exit_code"] to equal 0 + store refused as migration_cli_run of site_root and ["down", "--count", "1"] + expect refused["exit_code"] to equal 1 + expect refused["output"] contains "irreversible" to be yes + store unknown_target as migration_cli_run of site_root and ["up", "--target", "missing"] + expect unknown_target["exit_code"] to equal 1 + store invalid_count as migration_cli_run of site_root and ["down", "--count", "1.5"] + expect invalid_count["exit_code"] to equal 1 + store source_path as path_join of site_root and "app/migrations/20200102000000_session_csrf.wfl" + store original_source as test_read_text of source_path + call test_write_text with source_path and (original_source with "\n// edited immutable source\n") + store edited_status as migration_cli_run of site_root and ["status"] + expect edited_status["exit_code"] to equal 1 + expect edited_status["output"] contains "checksum" to be yes + expect file exists at (path_join of site_root and "scriptorium.db") to be no + store database_path as path_join of site_root and "configured-data/scriptorium.db" + open database at ("sqlite://" with database_path) as inspection_db + try: + store ledger_rows as query inspection_db with "SELECT * FROM _orm_migrations" + store event_rows as query inspection_db with "SELECT * FROM _orm_migration_events" + expect length of ledger_rows to equal 2 + expect length of event_rows to equal 2 + finally: + close database inspection_db + end try + finally: + call remove_dir with site_root and yes + end try + end test + + test "legacy plans respect their target and never stamp adoption history": + store site_root as migration_cli_fixture + try: + store database_path as path_join of site_root and "scriptorium.db" + open database at ("sqlite://" with database_path) as legacy_db + try: + store created as execute legacy_db with "CREATE TABLE sessions (id TEXT PRIMARY KEY, user_id INTEGER NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL)" + finally: + close database legacy_db + end try + store old_plan as migration_cli_run of site_root and ["plan", "--target", "20200101000000"] + expect old_plan["exit_code"] to equal 0 + expect old_plan["output"] contains "Pending after adoption: 0" to be yes + open database at ("sqlite://" with database_path) as current_legacy_db + try: + store altered as execute current_legacy_db with "ALTER TABLE sessions ADD COLUMN csrf_token TEXT NOT NULL DEFAULT ''" + finally: + close database current_legacy_db + end try + store impossible_plan as migration_cli_run of site_root and ["plan", "--target", "20200101000000"] + expect impossible_plan["exit_code"] to equal 1 + expect impossible_plan["output"] contains "predates the existing legacy schema" to be yes + open database at ("sqlite://" with database_path) as inspection_db + try: + store history_rows as query inspection_db with "SELECT name FROM sqlite_schema WHERE name='_orm_migrations'" + expect length of history_rows to equal 0 + finally: + close database inspection_db + end try + finally: + call remove_dir with site_root and yes + end try + end test + + test "scaffold instructions produce an executable registered up and down migration": + store site_root as migration_cli_fixture + try: + // Exercise an actual Windows-style checkout, including the registry + // line that this test edits after following scaffold instructions. + store original_bridge_path as path_join of site_root and "app/migrations.wfl" + store original_bridge as test_read_text of original_bridge_path + change original_bridge to replace "\r\n" with "\n" in original_bridge + change original_bridge to replace "\n" with "\r\n" in original_bridge + call test_write_text with original_bridge_path and original_bridge + store before_names as list_dir of (path_join of site_root and "app/migrations") + store scaffolded as migration_cli_run of site_root and ["new", "add_summary"] + expect scaffolded["exit_code"] to equal 0 + expect scaffolded["output"] contains "Append migration_" to be yes + expect scaffolded["output"] contains "empty scaffolds are rejected" to be yes + store after_names as list_dir of (path_join of site_root and "app/migrations") + expect length of after_names to equal ((length of before_names) plus 1) + expect file exists at (path_join of site_root and "scriptorium.db") to be no + store created_name as "" + for each version_name in after_names: + check if (contains of before_names and version_name) is equal to no: + change created_name to version_name + end check + end for + store created_id as substring of created_name and 0 and 14 + store created_path as path_join of site_root and "app/migrations" and created_name + store scaffold_text as test_read_text of created_path + store completed_text as replace "up_steps is []" with "up_steps is [(orm_migration_sql of \"INSERT INTO settings(skey,svalue) VALUES('scaffold-proof','applied')\" and [])]" in scaffold_text + change completed_text to replace "down_steps is []" with "down_steps is [(orm_migration_sql of \"DELETE FROM settings WHERE skey='scaffold-proof'\" and [])]" in completed_text + change completed_text to replace "TODO: describe why rollback cannot restore the data, or supply lossless down steps" with "" in completed_text + call test_write_text with created_path and completed_text + store bridge_path as path_join of site_root and "app/migrations.wfl" + store bridge_text as test_read_text of bridge_path + // The disposable registry edit must behave on CRLF Git checkouts. + change bridge_text to replace "\r\n" with "\n" in bridge_text + change bridge_text to replace "include from \"migrations/20200102000000_session_csrf.wfl\"" with ("include from \"migrations/" with created_name with "\"") in bridge_text + store registered_text as "" + for each source_line in (split bridge_text by "\n"): + store updated_line as source_line + check if source_line contains "return [(migration_20200101000000": + change updated_line to (substring of source_line and 0 and ((length of source_line) minus 1)) with ", (migration_" with created_id with " of (scriptorium_migration_source of \"" with created_name with "\"))]" + end check + change registered_text to registered_text with updated_line with "\n" + end for + call test_write_text with bridge_path and registered_text + store applied_scaffold as migration_cli_run of site_root and ["up"] + expect applied_scaffold["exit_code"] to equal 0 + expect applied_scaffold["output"] contains "Applied migrations: 3" to be yes + store rolled_scaffold as migration_cli_run of site_root and ["down", "--count", "1"] + expect rolled_scaffold["exit_code"] to equal 0 + expect rolled_scaffold["output"] contains "Applied migrations: 2" to be yes + store malicious_name as migration_cli_run of site_root and ["new", "../escape"] + expect malicious_name["exit_code"] to equal 1 + expect file exists at (path_join of site_root and "app/escape.wfl") to be no + finally: + call remove_dir with site_root and yes + end try + end test +end describe diff --git a/tests/tooling/runner.test.wfl b/tests/tooling/runner.test.wfl new file mode 100644 index 0000000..f17d49c --- /dev/null +++ b/tests/tooling/runner.test.wfl @@ -0,0 +1,177 @@ +include from "../../scripts/test_support.wfl" + +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_sources as path_join of script_directory and "fixtures" +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store artifact_root as path_join of repo_root and "target/test-artifacts/runner" and (generate_uuid) + +define action called runner_fixture: + store fixture_root as path_join of artifact_root and (generate_uuid) + call makedirs with (path_join of fixture_root and "scripts") + call makedirs with (path_join of fixture_root and "TestPrograms") + for each helper_name in ["run_tests.wfl", "test_support.wfl", "resolve_runtime.wfl", ".wflcfg"]: + call copy_file with (path_join of repo_root and "scripts" and helper_name) and (path_join of fixture_root and "scripts" and helper_name) + end for + call copy_file with (path_join of script_directory and ".wflcfg") and (path_join of fixture_root and ".wflcfg") + return fixture_root +end action + +define action called fixture_suite with parameters fixture_root and relative_path and behavior_name: + store destination as path_join of fixture_root and relative_path + call makedirs with (path_dirname of destination) + call copy_file with (path_join of fixture_sources and behavior_name) and destination +end action + +define action called invoke_runner with parameters fixture_root and extra_arguments: + store runner_path as path_join of fixture_root and "scripts/run_tests.wfl" + store invocation as [runner_path] + for each argument_value in extra_arguments: + push with invocation and argument_value + end for + store outcome as test_execute of runtime_path and invocation and artifact_root and 30 + display outcome["output"] + display outcome["error"] + return outcome +end action + +define action called runner_observations with parameters output_text: + store observations as [] + for each output_line in (string_split of output_text and newline): + check if starts_with of output_line and "RUN ": + store observed as parse_json of (substring of output_line and 4 and ((length of output_line) minus 4)) + push with observations and observed + end check + end for + return observations +end action + +describe "Complete WFL runner regressions": + teardown: + check if is_dir of artifact_root: + call remove_dir with artifact_root and yes + end check + end teardown + + test "nested suites are sorted and run from repository cwd": + store fixture_root as call runner_fixture + call fixture_suite with fixture_root and "TestPrograms/z.test.wfl" and "runner-pass.wfl" + call fixture_suite with fixture_root and "TestPrograms/nested/a.test.wfl" and "runner-pass.wfl" + call test_write_text with (path_join of fixture_root and "TestPrograms/example.wfl") and "invalid helper must not be discovered" + store outcome as invoke_runner of fixture_root and ["--group", "application"] + expect outcome["exit_code"] to equal 0 + store observed as runner_observations of outcome["output"] + expect (length of observed) to equal 2 + expect (path_basename of observed[0]["suite"]) to equal "a.test.wfl" + expect (path_basename of observed[1]["suite"]) to equal "z.test.wfl" + for each run_details in observed: + expect (replace "\\" with "/" in run_details["cwd"]) to equal (replace "\\" with "/" in fixture_root) + end for + end test + + test "real assertion and stderr failures preserve diagnostics and later suites run": + store fixture_root as call runner_fixture + call fixture_suite with fixture_root and "TestPrograms/a.test.wfl" and "runner-fail.wfl" + call fixture_suite with fixture_root and "TestPrograms/b.test.wfl" and "runner-error.wfl" + call fixture_suite with fixture_root and "TestPrograms/c.test.wfl" and "runner-pass.wfl" + store outcome as invoke_runner of fixture_root and ["--group", "application"] + expect outcome["exit_code"] to equal 1 + expect outcome["output"] to contain "intentional fixture stdout" + expect outcome["output"] to contain "intentional fixture failure" + expect outcome["output"] to contain "intentional runtime stdout" + expect outcome["output"] to contain "Division by zero" + expect outcome["output"] to contain "a.test.wfl" + expect outcome["output"] to contain "3 suites: 1 passed, 2 failed" + expect (length of (runner_observations of outcome["output"])) to equal 1 + end test + + test "timeout fails continues and terminates owned descendants": + store fixture_root as call runner_fixture + call fixture_suite with fixture_root and "TestPrograms/a.test.wfl" and "runner-timeout.wfl" + call fixture_suite with fixture_root and "TestPrograms/b.test.wfl" and "runner-pass.wfl" + call fixture_suite with fixture_root and "helpers/delayed-write.wfl" and "delayed-write.wfl" + store outcome as invoke_runner of fixture_root and ["--group", "application", "--timeout", "2"] + expect outcome["exit_code"] to equal 1 + expect (to_lowercase of outcome["output"]) to contain "timeout" + expect outcome["output"] to contain "diagnostic before runner timeout" + expect outcome["output"] to contain "2 suites: 1 passed, 1 failed" + expect (is_file of (path_join of fixture_root and "descendant-ready.txt")) to equal yes + wait for 3500 milliseconds + expect (path_exists of (path_join of fixture_root and "descendant-late.txt")) to equal no + end test + + test "empty suite discovery fails before execution": + store fixture_root as call runner_fixture + store outcome as invoke_runner of fixture_root and ["--group", "application"] + expect outcome["exit_code"] to equal 2 + expect outcome["output"] to contain "no WFL test suites" + end test + + test "missing interpreter fails before execution": + store fixture_root as call runner_fixture + call fixture_suite with fixture_root and "TestPrograms/one.test.wfl" and "runner-pass.wfl" + store missing_runtime as path_join of fixture_root and "absent-wfl" + store outcome as invoke_runner of fixture_root and ["--group", "application", "--wfl", missing_runtime] + expect outcome["exit_code"] to equal 2 + expect (length of (runner_observations of outcome["output"])) to equal 0 + expect outcome["output"] to contain "WFL executable" + end test + + test "nonpositive and nonfinite timeout values fail before execution": + store fixture_root as call runner_fixture + call fixture_suite with fixture_root and "TestPrograms/one.test.wfl" and "runner-pass.wfl" + for each invalid_timeout in ["0", "-1", "nan", "inf"]: + store outcome as invoke_runner of fixture_root and ["--group", "application", "--timeout", invalid_timeout] + expect outcome["exit_code"] to equal 2 + expect (length of (runner_observations of outcome["output"])) to equal 0 + expect outcome["output"] to contain "timeout must" + end for + end test + + test "missing requested Scribe source fails before local suites": + store fixture_root as call runner_fixture + call fixture_suite with fixture_root and "TestPrograms/one.test.wfl" and "runner-pass.wfl" + store outcome as invoke_runner of fixture_root and ["--group", "application", "--include-scribe"] + expect outcome["exit_code"] to equal 2 + expect outcome["output"] to contain "missing Scribe file" + expect (length of (runner_observations of outcome["output"])) to equal 0 + end test + + test "Scribe runs last in a removed disposable copy with a fresh build": + store fixture_root as call runner_fixture + call fixture_suite with fixture_root and "TestPrograms/one.test.wfl" and "runner-pass.wfl" + call test_write_text with (path_join of fixture_root and "lib/scribe/src/scribe.wfl") and "// synthetic source" + call fixture_suite with fixture_root and "lib/scribe/tests/scribe.test.wfl" and "runner-pass.wfl" + store original_build as path_join of fixture_root and "lib/scribe/build/existing.txt" + call test_write_text with original_build and "preserve" + call test_write_text with (path_join of fixture_root and "lib/scribe/.git") and "synthetic git pointer" + call test_write_text with (path_join of fixture_root and "lib/scribe/__pycache__/ignored.txt") and "stale cache" + store outcome as invoke_runner of fixture_root and ["--group", "application", "--include-scribe"] + expect outcome["exit_code"] to equal 0 + store observed as runner_observations of outcome["output"] + expect (length of observed) to equal 2 + expect observed[1]["build_exists"] to equal yes + expect observed[1]["stale_build"] to equal no + expect observed[1]["copied_git"] to equal no + expect observed[1]["copied_cache"] to equal no + expect (observed[1]["cwd"] is equal to (path_join of fixture_root and "lib/scribe")) to equal no + expect (path_exists of observed[1]["cwd"]) to equal no + expect (test_read_text of original_build) to equal "preserve" + end test + + test "default command discovers every maintained group and excludes helpers": + store fixture_root as call runner_fixture + for each group_path in ["TestPrograms", "tests/tooling", "tests/integration", "tests/runtime", "examples"]: + call fixture_suite with fixture_root and (group_path with "/nested/example.test.wfl") and "runner-pass.wfl" + call test_write_text with (path_join of fixture_root and group_path and "helper.wfl") and "must never execute" + end for + call test_write_text with (path_join of fixture_root and "lib/scribe/src/scribe.wfl") and "// synthetic source" + call fixture_suite with fixture_root and "lib/scribe/tests/scribe.test.wfl" and "runner-pass.wfl" + store outcome as invoke_runner of fixture_root and [] + expect outcome["exit_code"] to equal 0 + expect outcome["output"] to contain "6 suites: 6 passed, 0 failed" + expect (length of (runner_observations of outcome["output"])) to equal 6 + end test +end describe diff --git a/tests/tooling/test_repo_hygiene.py b/tests/tooling/test_repo_hygiene.py deleted file mode 100644 index 07bf7aa..0000000 --- a/tests/tooling/test_repo_hygiene.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Exercise the hygiene command against isolated Git checkouts, never this index.""" - -from pathlib import Path -import shutil -import subprocess -import sys -import tempfile -import tomllib -import unittest - - -PROJECT = Path(__file__).resolve().parents[2] -CHECKER = PROJECT / "scripts/check_repo_hygiene.py" -PROFILE = PROJECT / ".repo-hygiene.toml" - - -@unittest.skipUnless(shutil.which("git"), "Git is required for fixture checkouts") -class RepositoryHygieneTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory(prefix="scriptorium-hygiene-") - self.addCleanup(self.temp.cleanup) - self.root = Path(self.temp.name) - self.git("init", "--quiet") - self.profile_text = PROFILE.read_text("utf-8") - profile = tomllib.loads(self.profile_text) - for name in profile["required"]["files"]: - self.write(name, "Fixture file.\n") - self.write(".repo-hygiene.toml", self.profile_text) - self.write(".gitignore", "*.db\n*.private\n.env\n__pycache__/\n") - self.write("README.md", "[Architecture](docs/ARCHITECTURE.md)\n") - self.write("static/uploads/.gitkeep", "") - self.write("app/source.wfl", "display \"fixture\"\n") - self.git("add", ".") - # A gitlink records a commit object name. No remote, nested checkout, or - # object download is needed to test that the parent treats it opaquely. - self.git("update-index", "--add", "--cacheinfo", "160000", "1" * 40, "lib/scribe") - - def git(self, *args): - return subprocess.run( - ["git", "-C", str(self.root), *args], capture_output=True, check=True - ) - - def write(self, name, content): - path = self.root / name - path.parent.mkdir(parents=True, exist_ok=True) - if isinstance(content, bytes): - path.write_bytes(content) - else: - path.write_text(content, encoding="utf-8") - return path - - def run_checker(self): - return subprocess.run( - [sys.executable, str(CHECKER), "--root", str(self.root)], - capture_output=True, text=True, check=False, - ) - - def assert_result(self, code, fragment=None): - result = self.run_checker() - output = result.stdout + result.stderr - self.assertEqual(result.returncode, code, output) - if fragment is not None: - self.assertIn(fragment, output) - return output - - def test_clean_checkout_and_uninitialized_submodule_pass(self): - self.assert_result(0, "Repository hygiene passed") - - def test_new_untracked_root_file_is_checked_before_staging(self): - self.write("scratch.md", "Unapproved root note\n") - self.assert_result(1, "HYGIENE: root: scratch.md") - - def test_new_untracked_root_directory_is_rejected(self): - self.write("notes/design.md", "An unapproved root directory\n") - self.assert_result(1, "HYGIENE: root: notes/design.md") - - def test_legitimate_new_docs_and_binary_source_assets_pass(self): - self.write("docs/new-guide.md", "Maintained guide\n") - self.write("static/fonts/new.woff2", b"wOF2\x00fixture") - self.write("docs/screenshots/new.png", b"\x89PNG\r\n\x1a\n\x00fixture") - self.assert_result(0) - - def test_ignored_untracked_runtime_data_is_left_alone(self): - database = self.write("scriptorium.db", b"private runtime fixture") - self.write(".env", "TOKEN=fixture\n") - self.assert_result(0) - self.assertEqual(database.read_bytes(), b"private runtime fixture") - - def test_force_tracked_ignored_database_is_rejected(self): - self.write("app/fixture.db", b"runtime fixture") - self.git("add", "--force", "app/fixture.db") - self.assert_result(1, "HYGIENE: artifact: app/fixture.db") - - def test_repository_ignores_sqlite_databases_and_sidecars(self): - self.write(".gitignore", (PROJECT / ".gitignore").read_text("utf-8")) - for extension in ("db", "sqlite", "sqlite3"): - for suffix in ("", "-journal", "-wal", "-shm"): - self.write(f"site.{extension}{suffix}", b"local runtime fixture") - self.assert_result(0) - - def test_case_variants_and_nested_cache_are_rejected(self): - # Windows Git may ignore __PyCache__ under the lowercase ignore rule. - # Make these candidates visible on every platform for this check. - self.write(".gitignore", "# No ignores in this test.\n") - for name in ["app/site.SQLITE3-WAL", "docs/server.LOG", "scripts/__PyCache__/bad.txt"]: - with self.subTest(name=name): - self.write(name, "fixture\n") - self.assert_result(1, f"HYGIENE: artifact: {name}") - (self.root / name).unlink() - - def test_secret_names_are_rejected_at_any_depth(self): - for name in ["app/.env.production", "docs/id_rsa", "app/secret.KEY"]: - with self.subTest(name=name): - self.write(name, "fixture\n") - self.assert_result(1, f"HYGIENE: artifact: {name}") - (self.root / name).unlink() - - def test_upload_image_cannot_use_source_asset_exception(self): - self.write("static/uploads/photo.png", b"\x89PNG\x00runtime fixture") - self.assert_result(1, "HYGIENE: runtime-state: static/uploads/photo.png") - - def test_only_empty_declared_upload_placeholder_is_allowed(self): - self.write("static/uploads/.gitkeep", "runtime bytes\n") - self.assert_result(1, "HYGIENE: runtime-state: static/uploads/.gitkeep") - - def test_removed_and_empty_required_policy_fail(self): - (self.root / "GOVERNANCE.md").unlink() - self.assert_result(1, "HYGIENE: required: GOVERNANCE.md") - self.write("GOVERNANCE.md", "") - self.assert_result(1, "required file is empty") - - def test_ignored_required_policy_cannot_pass_by_existing_locally(self): - self.git("rm", "--cached", "--force", "GOVERNANCE.md") - self.write(".gitignore", "GOVERNANCE.md\n") - self.assert_result(1, "HYGIENE: required: GOVERNANCE.md") - - def test_worktree_link_changes_are_checked_without_staging(self): - self.write("README.md", "[Missing](docs/does-not-exist.md)\n") - self.assert_result(1, "local link is missing from candidate tree") - - def test_link_to_ignored_local_file_cannot_hide_broken_clone(self): - self.write("docs/local.private", "private fixture\n") - self.write("README.md", "[Local](docs/local.private)\n") - self.assert_result(1, "local link is missing from candidate tree") - - def test_links_to_new_candidates_directories_and_encoded_paths_pass(self): - self.write("docs/new file.md", "New guide\n") - self.write("README.md", "[Guide](docs/new%20file.md#title)\n[Docs](docs/)\n" - "[Spaced]()\n[Ref][guide]\n" - "[guide]: docs/new%20file.md \"A guide\"\n") - self.assert_result(0) - - def test_reference_and_image_link_destinations_are_checked(self): - self.write("README.md", "![Image](docs/missing.png)\n[ref]: docs/missing.md\n") - output = self.assert_result(1) - self.assertIn("docs/missing.png", output) - self.assertIn("docs/missing.md", output) - - def test_anchors_urls_fences_and_submodule_links_are_skipped(self): - self.write("README.md", "[Anchor](#unknown)\n[Web](https://example.invalid/nope)\n" - "[Mail](mailto:test@example.invalid)\n" - "[Scribe](lib/scribe/uninitialized.md)\n" - "```md\n[Example](missing.md)\n```\n" - "~~~md\n[Example](missing-too.md)\n~~~\n") - self.assert_result(0) - - def test_links_cannot_escape_checkout(self): - self.write("README.md", "[Outside](../outside.md)\n") - self.assert_result(1, "local link leaves checkout") - - def test_submodule_contents_are_not_recursively_inspected(self): - self.write("lib/scribe/private.db", "upstream runtime fixture\n") - self.write("lib/scribe/.env", "upstream fixture\n") - self.assert_result(0) - - def test_scribe_cannot_be_replaced_with_copied_files(self): - self.git("update-index", "--force-remove", "lib/scribe") - self.write("lib/scribe/src/scribe.wfl", "copied upstream fixture\n") - self.assert_result(1, "required Git gitlink is missing") - - def test_another_gitlink_cannot_bypass_artifact_inspection(self): - self.git("update-index", "--add", "--cacheinfo", "160000", "2" * 40, "lib/other") - self.assert_result(1, "HYGIENE: submodule: lib/other: unapproved gitlink") - - def test_indexed_symlink_is_rejected_even_when_checked_out_as_text(self): - target = self.write("app/linked.wfl", "../README.md") - blob = self.git("hash-object", "-w", str(target)).stdout.decode().strip() - self.git("update-index", "--add", "--cacheinfo", "120000", blob, "app/linked.wfl") - self.assert_result(1, "HYGIENE: file-type: app/linked.wfl") - - def test_malformed_profile_fails_closed(self): - self.write(".repo-hygiene.toml", "schema = [\n") - self.assert_result(2, "HYGIENE-ERROR") - - def test_missing_profile_fails_closed(self): - (self.root / ".repo-hygiene.toml").unlink() - self.assert_result(2, "cannot read .repo-hygiene.toml") - - def test_unknown_profile_key_fails_closed(self): - self.write(".repo-hygiene.toml", self.profile_text.replace( - "allowed-dirs =", "allowed-directories =" - )) - self.assert_result(2, "missing or unknown keys") - - def test_profile_cannot_use_escape_paths(self): - self.write(".repo-hygiene.toml", self.profile_text.replace( - '"docs/ARCHITECTURE.md"', '"../ARCHITECTURE.md"' - )) - self.assert_result(2, "non-relative path") - - def test_git_failure_fails_closed(self): - # Rename only this fixture's .git, without touching the actual checkout. - (self.root / ".git").rename(self.root / "fixture-index-away") - self.assert_result(2, "HYGIENE-ERROR: git") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/tooling/test_run_tests.py b/tests/tooling/test_run_tests.py deleted file mode 100644 index 077e9ff..0000000 --- a/tests/tooling/test_run_tests.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Exercise the test runner as a process with a controlled interpreter.""" - -import json -import os -from pathlib import Path -import shutil -import subprocess -import sys -import tempfile -import unittest - - -RUNNER = Path(__file__).resolve().parents[2] / "scripts" / "run_tests.py" - - -class RunTestsTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory(prefix="scriptorium-runner-test-") - self.addCleanup(self.temp.cleanup) - self.root = Path(self.temp.name) - (self.root / "scripts").mkdir() - shutil.copy2(RUNNER, self.root / "scripts" / "run_tests.py") - (self.root / "TestPrograms").mkdir() - fake = self.root / "fake_wfl.py" - fake.write_text( - "import json, pathlib, sys, time\n" - "suite = pathlib.Path(sys.argv[2])\n" - "case = json.loads(suite.read_text())\n" - "print('RUN ' + json.dumps({'suite': str(suite), " - "'cwd': str(pathlib.Path.cwd()), 'build_exists': pathlib.Path('build').is_dir()}), flush=True)\n" - "if case.get('stderr'): print(case['stderr'], file=sys.stderr, flush=True)\n" - "time.sleep(case.get('sleep', 0))\n" - "sys.exit(case.get('exit', 0))\n", - encoding="utf-8", - ) - if os.name == "nt": - self.interpreter = self.root / "fake-wfl.cmd" - self.interpreter.write_text( - f'@echo off\n"{sys.executable}" "{fake}" %*\n', encoding="utf-8" - ) - else: - self.interpreter = self.root / "fake-wfl" - self.interpreter.write_text( - f"#!{sys.executable}\n" + fake.read_text(encoding="utf-8"), - encoding="utf-8", - ) - self.interpreter.chmod(0o755) - - def suite(self, name="one.test.wfl", **behavior): - path = self.root / "TestPrograms" / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(behavior), encoding="utf-8") - return path - - def run_runner(self, *args): - return subprocess.run( - [sys.executable, str(self.root / "scripts" / "run_tests.py"), - "--wfl", str(self.interpreter), *args], - cwd=self.root.parent, - text=True, - capture_output=True, - timeout=15, - ) - - @staticmethod - def runs(result): - return [json.loads(line[4:]) for line in result.stdout.splitlines() - if line.startswith("RUN ")] - - def test_discovers_nested_suites_in_order_and_uses_repository_cwd(self): - self.suite("z.test.wfl") - self.suite("nested/a.test.wfl") - (self.root / "TestPrograms" / "example.wfl").write_text("not a suite") - result = self.run_runner() - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - runs = self.runs(result) - self.assertEqual([Path(run["suite"]).name for run in runs], - ["a.test.wfl", "z.test.wfl"]) - # Windows TEMP can use a DOS short path that the runner resolves to its - # long spelling. Assert directory identity, not how the path is spelled. - self.assertTrue(all(Path(run["cwd"]).samefile(self.root) for run in runs), - f"Expected {self.root}, got {[run['cwd'] for run in runs]}") - - def test_failing_suite_preserves_diagnostics_and_later_suite_runs(self): - self.suite("a.test.wfl", exit=7, stderr="intentional fixture failure") - self.suite("b.test.wfl") - result = self.run_runner() - self.assertEqual(result.returncode, 1, result.stdout + result.stderr) - self.assertEqual(len(self.runs(result)), 2) - self.assertIn("intentional fixture failure", result.stderr) - self.assertIn("a.test.wfl", result.stdout + result.stderr) - - def test_timeout_fails_and_later_suite_still_runs(self): - self.suite("a.test.wfl", sleep=2) - self.suite("b.test.wfl") - result = self.run_runner("--timeout", "1") - self.assertEqual(result.returncode, 1, result.stdout + result.stderr) - self.assertEqual(len(self.runs(result)), 2) - self.assertIn("timeout", (result.stdout + result.stderr).lower()) - - def test_empty_suite_set_is_an_error(self): - result = self.run_runner() - self.assertNotEqual(result.returncode, 0) - self.assertIn("no", (result.stdout + result.stderr).lower()) - - def test_missing_interpreter_is_an_error(self): - self.suite() - result = self.run_runner("--wfl", str(self.root / "absent-wfl")) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(self.runs(result), []) - - def test_nonpositive_and_nonfinite_timeout_are_rejected(self): - self.suite() - for value in ("0", "-1", "nan", "inf"): - with self.subTest(value=value): - result = self.run_runner("--timeout", value) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(self.runs(result), []) - - def test_missing_scribe_source_is_an_error(self): - self.suite() - result = self.run_runner("--include-scribe") - self.assertNotEqual(result.returncode, 0) - self.assertIn("scribe", (result.stdout + result.stderr).lower()) - - def test_upstream_suite_runs_in_disposable_copy_with_build_directory(self): - self.suite() - scribe = self.root / "lib" / "scribe" - (scribe / "src").mkdir(parents=True) - (scribe / "src" / "scribe.wfl").write_text("fixture") - (scribe / "tests").mkdir() - (scribe / "tests" / "scribe.test.wfl").write_text("{}") - (scribe / "build").mkdir() - (scribe / "build" / "existing.txt").write_text("preserve") - result = self.run_runner("--include-scribe") - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - runs = self.runs(result) - self.assertEqual(len(runs), 2) - upstream = Path(runs[-1]["cwd"]) - self.assertNotEqual(upstream, scribe) - self.assertTrue(runs[-1]["build_exists"]) - self.assertFalse(upstream.exists(), "temporary Scribe copy was not removed") - self.assertEqual((scribe / "build" / "existing.txt").read_text(), "preserve") - - -if __name__ == "__main__": - unittest.main()