From 84cb272cabce9dae9375359a1b56ed751fe5e3e0 Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 06:39:26 -0500 Subject: [PATCH 1/9] test: record explicit CLI execution budget regressions --- .../2026-09-20-cli-execution-budget.md | 56 +++++++++++++++++ TestPrograms/cli_budget/.wflcfg | 7 +++ TestPrograms/cli_budget/arguments.test.wfl | 63 +++++++++++++++++++ TestPrograms/cli_budget/deadlines.test.wfl | 61 ++++++++++++++++++ .../cli_budget/server-policy.test.wfl | 39 ++++++++++++ tests/fixtures/cli_budget/.wflcfg | 6 ++ tests/fixtures/cli_budget/arguments.wfl | 3 + tests/fixtures/cli_budget/child-config.wfl | 6 ++ tests/fixtures/cli_budget/late-marker.wfl | 4 ++ tests/fixtures/cli_budget/long-run.test.wfl | 14 +++++ tests/fixtures/cli_budget/marker.wfl | 4 ++ tests/fixtures/cli_budget/owned-child.wfl | 6 ++ tests/fixtures/cli_budget/server-client.wfl | 9 +++ tests/fixtures/cli_budget/shared-child.wfl | 5 ++ tests/fixtures/cli_budget/shared-parent.wfl | 3 + tests/fixtures/cli_budget/slow-peer.wfl | 12 ++++ tests/fixtures/cli_budget/test-mode.test.wfl | 6 ++ tests/fixtures/cli_budget/wait.wfl | 7 +++ 18 files changed, 311 insertions(+) create mode 100644 Engineering/evidence/2026-09-20-cli-execution-budget.md create mode 100644 TestPrograms/cli_budget/.wflcfg create mode 100644 TestPrograms/cli_budget/arguments.test.wfl create mode 100644 TestPrograms/cli_budget/deadlines.test.wfl create mode 100644 TestPrograms/cli_budget/server-policy.test.wfl create mode 100644 tests/fixtures/cli_budget/.wflcfg create mode 100644 tests/fixtures/cli_budget/arguments.wfl create mode 100644 tests/fixtures/cli_budget/child-config.wfl create mode 100644 tests/fixtures/cli_budget/late-marker.wfl create mode 100644 tests/fixtures/cli_budget/long-run.test.wfl create mode 100644 tests/fixtures/cli_budget/marker.wfl create mode 100644 tests/fixtures/cli_budget/owned-child.wfl create mode 100644 tests/fixtures/cli_budget/server-client.wfl create mode 100644 tests/fixtures/cli_budget/shared-child.wfl create mode 100644 tests/fixtures/cli_budget/shared-parent.wfl create mode 100644 tests/fixtures/cli_budget/slow-peer.wfl create mode 100644 tests/fixtures/cli_budget/test-mode.test.wfl create mode 100644 tests/fixtures/cli_budget/wait.wfl diff --git a/Engineering/evidence/2026-09-20-cli-execution-budget.md b/Engineering/evidence/2026-09-20-cli-execution-budget.md new file mode 100644 index 00000000..424234ca --- /dev/null +++ b/Engineering/evidence/2026-09-20-cli-execution-budget.md @@ -0,0 +1,56 @@ +# Finite invocation execution budgets + +Risk class: **R3** (resource limits, CLI compatibility and subprocess lifecycle). + +The Scriptorium full Linux suite reached the existing CLI's 300-second execution +deadline on official WFL 26.9.14 after about 34 suites, then subsequent child +launches failed against the same expired budget. The configured runner timeout +could not extend this limit. The same published source completed the Windows +consumer suite in about 160 seconds. This is a real capacity failure, not a +reason to hide suite failures or exempt a batch runner with `main loop`. + +The published source `8d82d785ea59300834de1c48b04a7ba0e187a1cd` +unconditionally applies `timeout_seconds.min(300)` in `src/main.rs`. +Configuration accepts larger values, but there is no existing file-entrypoint +override. The July 10 bind-address diary records retaining the historical cap +for compatibility. The new option leaves that default/config behavior intact. + +## Red evidence + +New scenarios, fixtures and drivers are WFL. Before implementation, the three +`TestPrograms/cli_budget/*.test.wfl` programs were run with the official Windows +26.9.14 executable (SHA-256 +`da109d5926f6af4a2f24c45150140764c406c055aef2dd7e43e45b85278f1dfe`). +The argument suite failed 5/5 expectations, deadlines passed its unchanged +configuration baseline and failed the four override cases, and server-policy +failed its one override case. All programs parsed and ran as WFL tests. The new +flag is unavailable on that release; these are CLI capability failures, not +claims that the old runtime implemented the new flag incorrectly. The existing +consumer 300-second failure establishes the underlying semantic limitation. + +The initial draft had an invalid reserved variable name and expected the wrong +timeout wording. Those fixture mistakes were corrected before the recorded Red +run; they are not counted as product failures. Logs are retained only under +ignored `target/reports/cli-budget/red-*.log`. + +## Acceptance and design + +`--execution-timeout SECONDS`, before the source filename, selects one finite +invocation deadline from 1 through 31,536,000 whole seconds. It changes only +`BudgetLimits.max_duration` before the shared budget starts. It does not change +`WflConfig`, default/config caps, request/stream limits, explicit subprocess wait +timeouts, subprocess permissions, operation/depth/size ceilings, or separately +launched WFL child configuration. Ordinary foreground operations continue to +share the invocation deadline; their duration therefore grows with an explicit +larger invocation budget. Included and executed files share that same deadline. + +Fast WFL suites cover operands, routing, test mode, cumulative deadlines, child +configuration, parent-expiration cleanup, and unchanged main-loop HTTP timeout. +The cleanup test first proves the child can write under its own longer budget. +`tests/fixtures/cli_budget/long-run.test.wfl` is a real 305-second ordinary run, +to be invoked explicitly by both integration jobs with a 330-second budget. +It belongs outside the recursive 30-second program sweep and must not be skipped. + +Green verification, independent review and exact-head CI will be recorded before +the change is submitted as ready to merge. No merge or release is authorized by +this evidence record itself. diff --git a/TestPrograms/cli_budget/.wflcfg b/TestPrograms/cli_budget/.wflcfg new file mode 100644 index 00000000..39c47274 --- /dev/null +++ b/TestPrograms/cli_budget/.wflcfg @@ -0,0 +1,7 @@ +# Trusted CLI conformance drivers own only synthetic child programs. +timeout_seconds = 60 +execution_logging = false +debug_report_enabled = false +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true diff --git a/TestPrograms/cli_budget/arguments.test.wfl b/TestPrograms/cli_budget/arguments.test.wfl new file mode 100644 index 00000000..e00482a8 --- /dev/null +++ b/TestPrograms/cli_budget/arguments.test.wfl @@ -0,0 +1,63 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" +store case_root as path_join of repo_root and "target/test-artifacts/cli-budget-arguments" and (generate_uuid) +call makedirs with case_root + +describe "Explicit execution budget CLI arguments": + teardown: + call remove_dir with case_root and yes + end teardown + + test "the finite maximum is accepted without changing script arguments": + store source_path as path_join of fixture_root and "arguments.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "31536000", source_path, "literal spaces", "--execution-timeout", "0"] as outcome + expect outcome["exit_code"] to equal 0 + expect outcome["output"] to contain "ARG:literal spaces" + expect outcome["output"] to contain "ARG:--execution-timeout" + expect outcome["output"] to contain "ARG:0" + end test + + test "invalid operands fail before executing the source": + store source_path as path_join of fixture_root and "marker.wfl" + store marker_path as path_join of case_root and "must-not-exist.txt" + for each invalid_value in ["0", "-1", "1.5", "nan", "inf", "31536001", "18446744073709551616", "", "+1"]: + wait for execute command runtime_path with arguments ["--execution-timeout", invalid_value, source_path, marker_path] as outcome + expect outcome["exit_code"] to equal 2 + expect outcome["error"] to contain "--execution-timeout requires a whole number of seconds from 1 to 31536000" + expect (is_file of marker_path) to equal no + end for + end test + + test "missing operands and duplicate overrides are actionable errors": + wait for execute command runtime_path with arguments ["--execution-timeout"] as missing_outcome + expect missing_outcome["exit_code"] to equal 2 + expect missing_outcome["error"] to contain "--execution-timeout requires a whole number" + store source_path as path_join of fixture_root and "arguments.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "2", "--execution-timeout", "3", source_path] as duplicate + expect duplicate["exit_code"] to equal 2 + expect duplicate["error"] to contain "--execution-timeout may be specified only once" + end test + + test "test mode works in either prefix option order": + store source_path as path_join of fixture_root and "test-mode.test.wfl" + for each option_list in [["--test", "--execution-timeout", "3", source_path], ["--execution-timeout", "3", "--test", source_path]]: + wait for execute command runtime_path with arguments option_list as outcome + expect outcome["exit_code"] to equal 0 + expect outcome["output"] to contain "TEST MODE EXECUTED" + expect outcome["output"] to contain "Failed: 0" + end for + end test + + test "source-consuming modes reject an override after the filename": + store source_path as path_join of fixture_root and "arguments.wfl" + for each mode_option in ["--analyze", "--lint"]: + wait for execute command runtime_path with arguments [mode_option, source_path, "--execution-timeout", "3"] as outcome + expect outcome["exit_code"] to equal 2 + expect outcome["error"] to contain "--execution-timeout must appear before the source filename" + end for + end test +end describe diff --git a/TestPrograms/cli_budget/deadlines.test.wfl b/TestPrograms/cli_budget/deadlines.test.wfl new file mode 100644 index 00000000..a34a41cb --- /dev/null +++ b/TestPrograms/cli_budget/deadlines.test.wfl @@ -0,0 +1,61 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" +store case_root as path_join of repo_root and "target/test-artifacts/cli-budget-deadlines" and (generate_uuid) +call makedirs with case_root + +describe "Invocation-only execution deadlines": + teardown: + call remove_dir with case_root and yes + end teardown + + test "configuration still governs ordinary invocations": + store source_path as path_join of fixture_root and "wait.wfl" + wait for execute command runtime_path with arguments [source_path, "1500"] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "COMPLETED") to equal no + end test + + test "an explicit budget extends this run beyond its configuration": + store source_path as path_join of fixture_root and "wait.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "4", source_path, "1500"] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "COMPLETED" + end test + + test "executed files share the cumulative invocation deadline": + store source_path as path_join of fixture_root and "shared-parent.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT COMPLETE") to equal no + end test + + test "expiration closes an owned child before it can write later": + store late_path as path_join of fixture_root and "late-marker.wfl" + store control_path as path_join of case_root and "control.txt" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", late_path, control_path] as control_outcome + expect control_outcome["success"] to equal yes + expect (is_file of control_path) to equal yes + store source_path as path_join of fixture_root and "owned-child.wfl" + store marker_path as path_join of case_root and "late.txt" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path, marker_path] as outcome + expect outcome["success"] to equal no + expect outcome["output"] to contain "OWNED CHILD STARTED" + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT COMPLETE") to equal no + wait for 3000 milliseconds + expect (is_file of marker_path) to equal no + end test + + test "separate WFL children retain their own configuration deadline": + store source_path as path_join of fixture_root and "child-config.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", source_path] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "CHILD CONFIGURATION RETAINED" + end test +end describe diff --git a/TestPrograms/cli_budget/server-policy.test.wfl b/TestPrograms/cli_budget/server-policy.test.wfl new file mode 100644 index 00000000..eba256ff --- /dev/null +++ b/TestPrograms/cli_budget/server-policy.test.wfl @@ -0,0 +1,39 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" + +describe "Execution override preserves server operation limits": + test "a main-loop request retains the configured one-second timeout": + store peer_path as path_join of fixture_root and "slow-peer.wfl" + wait for spawn command runtime_path with arguments [peer_path] as peer_process + try: + store startup_text as "" + store peer_url as "" + count from 1 to 200: + wait for read output from process peer_process as fresh_text + change startup_text to startup_text with fresh_text + for each output_line in (split startup_text by "\n"): + check if output_line starts with "READY ": + change peer_url to trim of (substring of output_line and 6 and ((length of output_line) minus 6)) + end check + end for + check if peer_url is not equal to "": + break + end check + wait for 10 milliseconds + end count + expect peer_url is not equal to "" to equal yes + store client_path as path_join of fixture_root and "server-client.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", client_path, peer_url] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT RECEIVE") to equal no + finally: + close process peer_process + end try + expect (process peer_process is running) to equal no + end test +end describe diff --git a/tests/fixtures/cli_budget/.wflcfg b/tests/fixtures/cli_budget/.wflcfg new file mode 100644 index 00000000..2a95c9c7 --- /dev/null +++ b/tests/fixtures/cli_budget/.wflcfg @@ -0,0 +1,6 @@ +timeout_seconds = 1 +execution_logging = false +debug_report_enabled = false +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true diff --git a/tests/fixtures/cli_budget/arguments.wfl b/tests/fixtures/cli_budget/arguments.wfl new file mode 100644 index 00000000..2a5c9a4e --- /dev/null +++ b/tests/fixtures/cli_budget/arguments.wfl @@ -0,0 +1,3 @@ +for each argument_value in args: + display "ARG:" with argument_value +end for diff --git a/tests/fixtures/cli_budget/child-config.wfl b/tests/fixtures/cli_budget/child-config.wfl new file mode 100644 index 00000000..758dafee --- /dev/null +++ b/tests/fixtures/cli_budget/child-config.wfl @@ -0,0 +1,6 @@ +store runtime_path as call current_executable +store child_path as path_join of script_directory and "wait.wfl" +wait for execute command runtime_path with arguments [child_path, "1500"] as outcome +expect outcome["success"] to equal no +expect outcome["error"] to contain "timeout (1s)" +display "CHILD CONFIGURATION RETAINED" diff --git a/tests/fixtures/cli_budget/late-marker.wfl b/tests/fixtures/cli_budget/late-marker.wfl new file mode 100644 index 00000000..e81bb8ed --- /dev/null +++ b/tests/fixtures/cli_budget/late-marker.wfl @@ -0,0 +1,4 @@ +wait for 2500 milliseconds +open file at args[0] for writing as marker_file +wait for write content "late child survived" into marker_file +close file marker_file diff --git a/tests/fixtures/cli_budget/long-run.test.wfl b/tests/fixtures/cli_budget/long-run.test.wfl new file mode 100644 index 00000000..76faa4e0 --- /dev/null +++ b/tests/fixtures/cli_budget/long-run.test.wfl @@ -0,0 +1,14 @@ +// Invoked explicitly by both integration jobs with --execution-timeout 330. +// This real boundary test must not enter a lifetime-exempt main loop. +describe "Explicit execution budget above five minutes": + test "ordinary WFL execution can complete beyond the legacy cap": + store started_at as current time in milliseconds + wait for 305 seconds + count from 1 to 5000: + store observed as count + end count + store elapsed_ms as (current time in milliseconds) minus started_at + expect elapsed_ms to be greater than 300000 + display "LONG EXECUTION BUDGET VERIFIED" + end test +end describe diff --git a/tests/fixtures/cli_budget/marker.wfl b/tests/fixtures/cli_budget/marker.wfl new file mode 100644 index 00000000..d4117279 --- /dev/null +++ b/tests/fixtures/cli_budget/marker.wfl @@ -0,0 +1,4 @@ +open file at args[0] for writing as marker_file +wait for write content "executed" into marker_file +close file marker_file +display "MARKER WRITTEN" diff --git a/tests/fixtures/cli_budget/owned-child.wfl b/tests/fixtures/cli_budget/owned-child.wfl new file mode 100644 index 00000000..787a393c --- /dev/null +++ b/tests/fixtures/cli_budget/owned-child.wfl @@ -0,0 +1,6 @@ +store runtime_path as call current_executable +store child_path as path_join of script_directory and "late-marker.wfl" +wait for spawn command runtime_path with arguments ["--execution-timeout", "10", child_path, args[0]] as owned_child +display "OWNED CHILD STARTED" +wait for process owned_child to complete with timeout 10 and read result as outcome +display "MUST NOT COMPLETE OWNED WAIT" diff --git a/tests/fixtures/cli_budget/server-client.wfl b/tests/fixtures/cli_budget/server-client.wfl new file mode 100644 index 00000000..9dc4c7ae --- /dev/null +++ b/tests/fixtures/cli_budget/server-client.wfl @@ -0,0 +1,9 @@ +main loop: + try: + open url at args[0] and read content as content_value + display "MUST NOT RECEIVE LATE BODY" + when error: + display error_message + end try + break +end loop diff --git a/tests/fixtures/cli_budget/shared-child.wfl b/tests/fixtures/cli_budget/shared-child.wfl new file mode 100644 index 00000000..f729500a --- /dev/null +++ b/tests/fixtures/cli_budget/shared-child.wfl @@ -0,0 +1,5 @@ +wait for 700 milliseconds +count from 1 to 5000: + store observed as count +end count +display "MUST NOT COMPLETE CHILD" diff --git a/tests/fixtures/cli_budget/shared-parent.wfl b/tests/fixtures/cli_budget/shared-parent.wfl new file mode 100644 index 00000000..c0e22415 --- /dev/null +++ b/tests/fixtures/cli_budget/shared-parent.wfl @@ -0,0 +1,3 @@ +wait for 700 milliseconds +execute file "shared-child.wfl" +display "MUST NOT COMPLETE SHARED RUN" diff --git a/tests/fixtures/cli_budget/slow-peer.wfl b/tests/fixtures/cli_budget/slow-peer.wfl new file mode 100644 index 00000000..3dcdde35 --- /dev/null +++ b/tests/fixtures/cli_budget/slow-peer.wfl @@ -0,0 +1,12 @@ +listen on port 0 as slow_peer +store listener_text as "" with slow_peer +display "READY http://" with (substring of listener_text and 11 and ((length of listener_text) minus 11)) +main loop: + wait for request comes in on slow_peer as incoming + wait for 2500 milliseconds + try: + respond to incoming with "late response" + when error: + display "TIMED CLIENT DISCONNECTED" + end try +end loop diff --git a/tests/fixtures/cli_budget/test-mode.test.wfl b/tests/fixtures/cli_budget/test-mode.test.wfl new file mode 100644 index 00000000..16ff54ef --- /dev/null +++ b/tests/fixtures/cli_budget/test-mode.test.wfl @@ -0,0 +1,6 @@ +describe "Execution budget preserves test mode": + test "assertions actually execute": + expect 7 to equal 7 + display "TEST MODE EXECUTED" + end test +end describe diff --git a/tests/fixtures/cli_budget/wait.wfl b/tests/fixtures/cli_budget/wait.wfl new file mode 100644 index 00000000..9a665662 --- /dev/null +++ b/tests/fixtures/cli_budget/wait.wfl @@ -0,0 +1,7 @@ +store delay_ms as parse_json of args[0] +wait for delay_ms milliseconds +// Exercise deadline checkpoints after sleeping, before claiming completion. +count from 1 to 5000: + store observed as count +end count +display "COMPLETED" From 68c466504610cf5bfff26e8755f3b6064ceab2a2 Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 06:54:17 -0500 Subject: [PATCH 2/9] feat(cli): allow explicit finite invocation execution budgets --- .github/workflows/ci.yml | 7 ++ CLAUDE.md | 1 + Docs/reference/configuration-reference.md | 31 ++++++++ .../2026-09-20-cli-execution-budget.md | 70 +++++++++++++++++-- .../2026/2026-09-20-cli-execution-budget.md | 27 +++++++ README.md | 1 + TestPrograms/cli_budget/arguments.test.wfl | 9 +++ TestPrograms/cli_budget/deadlines.test.wfl | 2 +- .../cli_budget/resource-policy.test.wfl | 32 +++++++++ src/main.rs | 57 ++++++++++++--- testing.md | 9 +++ tests/fixtures/cli_budget/denied/.wflcfg | 4 ++ tests/fixtures/cli_budget/denied/process.wfl | 3 + .../fixtures/cli_budget/longer-config/.wflcfg | 3 + .../cli_budget/longer-config/wait.wfl | 1 + tests/fixtures/cli_budget/operations/.wflcfg | 4 ++ tests/fixtures/cli_budget/operations/loop.wfl | 4 ++ tests/fixtures/cli_budget/shared-parent.wfl | 2 +- 18 files changed, 249 insertions(+), 18 deletions(-) create mode 100644 History/dev-diary/2026/2026-09-20-cli-execution-budget.md create mode 100644 TestPrograms/cli_budget/resource-policy.test.wfl create mode 100644 tests/fixtures/cli_budget/denied/.wflcfg create mode 100644 tests/fixtures/cli_budget/denied/process.wfl create mode 100644 tests/fixtures/cli_budget/longer-config/.wflcfg create mode 100644 tests/fixtures/cli_budget/longer-config/wait.wfl create mode 100644 tests/fixtures/cli_budget/operations/.wflcfg create mode 100644 tests/fixtures/cli_budget/operations/loop.wfl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1997465..79343ef7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -313,6 +313,13 @@ jobs: RUST_MIN_STACK: '8388608' run: ./scripts/run_integration_tests.ps1 + # Real ordinary execution beyond the historical 300-second CLI ceiling. + # The complete scenario/assertions are WFL; this step only invokes it. + # It cannot use the recursive program sweep's 30-second per-file bound. + - name: Verify explicit execution budget beyond five minutes + timeout-minutes: 6 + run: ./target/release/wfl --execution-timeout 330 --test tests/fixtures/cli_budget/long-run.test.wfl + # Validate that documentation examples still parse/analyze/lint against the # current release binary (testing.md requires docs validation in CI). # `--force` ignores the committed cache so CI always re-validates rather diff --git a/CLAUDE.md b/CLAUDE.md index 25355713..869ee11b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,6 +146,7 @@ Source Code → Lexer → Parser → Analyzer → Type Checker → Interpreter - `wfl --dump-env`: Dump environment for troubleshooting. - `wfl --analyze `: Run static analysis. - `wfl --test `: Run file in test mode (executes describe/test blocks). +- `wfl --execution-timeout `: Override only this invocation's shared deadline; whole seconds from 1 through 31536000, before the filename. Defaults, config caps and other limits remain unchanged. ## Key Language Features - **Natural Language Syntax**: `store name as "value"`, `check if x is greater than 5`. diff --git a/Docs/reference/configuration-reference.md b/Docs/reference/configuration-reference.md index 338f215e..96e0b921 100644 --- a/Docs/reference/configuration-reference.md +++ b/Docs/reference/configuration-reference.md @@ -290,6 +290,37 @@ so a stalled remote peer cannot wedge the server indefinitely. - **Default:** `60` - **Example:** `timeout_seconds = 300` +The file CLI preserves a maximum of 300 seconds for this configuration setting. +For a trusted batch job that needs a longer finite invocation, place +`--execution-timeout SECONDS` before its filename: + +```bash +wfl --execution-timeout 1200 batch.wfl +wfl --execution-timeout 1200 --test batch.test.wfl +``` + +The option accepts whole seconds from 1 through 31,536,000 (one year). Zero, +negative or fractional values, nonnumeric values and duplicate options are +errors; there is no unlimited value. Without the option, the default remains +60 seconds and configured values retain the 300-second cap. The option may also +bound source analysis, lexing and parsing. It cannot accompany configuration +maintenance, environment dumps or editor launch. After an executable source +filename, arguments are passed literally to that program instead. + +This is an invocation-only override of the shared execution deadline, starting +before source loading and covering the front end, interpreter, includes and +executed files. It does not reset between tests or nested files, and it does not +implicitly change the configuration of separately launched WFL children. +Ordinary foreground subprocess operations continue sharing the invocation +deadline, while explicit process-wait timeouts remain independently enforced. + +The override does not change other resource limits, subprocess permissions, +request/stream limits, or configured per-operation timeouts. A server's `main +loop` retains its existing lifetime exemption; its outbound requests and +implicit subprocess waits keep their finite configured timeouts. Use the option +only when the caller deliberately grants the job more execution time; it does +not establish a sandbox for untrusted programs. + #### `logging_enabled` Enables logging output to `wfl.log` in the script’s directory. diff --git a/Engineering/evidence/2026-09-20-cli-execution-budget.md b/Engineering/evidence/2026-09-20-cli-execution-budget.md index 424234ca..7b01049a 100644 --- a/Engineering/evidence/2026-09-20-cli-execution-budget.md +++ b/Engineering/evidence/2026-09-20-cli-execution-budget.md @@ -5,9 +5,14 @@ Risk class: **R3** (resource limits, CLI compatibility and subprocess lifecycle) The Scriptorium full Linux suite reached the existing CLI's 300-second execution deadline on official WFL 26.9.14 after about 34 suites, then subsequent child launches failed against the same expired budget. The configured runner timeout -could not extend this limit. The same published source completed the Windows -consumer suite in about 160 seconds. This is a real capacity failure, not a +could not extend this limit. The same published runtime completed the Windows +consumer suite at `07e8adcb` in about 160 seconds: 41 suites passed and its one +deliberately failing gate test correctly made the invocation exit 1. That local +consumer log is `target/full-official-windows-red.log` in Scriptorium. This is a +real capacity failure, not a reason to hide suite failures or exempt a batch runner with `main loop`. +The consumer Red is [Scriptorium CI35507634634](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35507634634) +at `07e8adcb`, using the published runtime rather than a development build. The published source `8d82d785ea59300834de1c48b04a7ba0e187a1cd` unconditionally applies `timeout_seconds.min(300)` in `src/main.rs`. @@ -45,12 +50,63 @@ share the invocation deadline; their duration therefore grows with an explicit larger invocation budget. Included and executed files share that same deadline. Fast WFL suites cover operands, routing, test mode, cumulative deadlines, child -configuration, parent-expiration cleanup, and unchanged main-loop HTTP timeout. +configuration, parent-expiration cleanup, and unchanged main-loop HTTP timeout, +operation limits and subprocess permissions. The cleanup test first proves the child can write under its own longer budget. `tests/fixtures/cli_budget/long-run.test.wfl` is a real 305-second ordinary run, -to be invoked explicitly by both integration jobs with a 330-second budget. +invoked explicitly by both integration jobs with a 330-second budget. It belongs outside the recursive 30-second program sweep and must not be skipped. -Green verification, independent review and exact-head CI will be recorded before -the change is submitted as ready to merge. No merge or release is authorized by -this evidence record itself. +## Green verification and review + +Red commit `84cb272c` preserves the original executable capability tests before +the product implementation. Green adds the CLI parsing and budget selection in +`src/main.rs`; no interpreter or configuration policy implementation changes. +Further cases cover disallowed CLI modes and unchanged operation/permission +limits. While bringing the fixtures to Green, the shared-file fixture gained +the required `execute file at` syntax and the child assertion fixture gained +`--test`. These fixture corrections are not product defects or semantic Red +evidence. The later resource-policy cases also fail against the official runtime +because the flag is absent, rather than because the old policies are incorrect. + +The release candidate reports version 26.9.15 and has SHA-256 +`1185f150c8214d982a27431d4b88b94f7f20d6ce6c718161c35120deb817650f`. +Local Windows verification at the reviewed source: + +- `cargo build --release --locked`: passed. +- Four fast WFL suites: arguments 6/6, deadlines 5/5, server policy 1/1, + resource policy 3/3; all 15 passed. +- `wfl --execution-timeout 330 --test tests/fixtures/cli_budget/long-run.test.wfl`: + passed 1/1 after the actual 305-second wait and checkpoint. This run was not + shortened, skipped or made lifetime-exempt. +- Existing Rust compatibility suites: 139/139 passed across execution budget, + CLI help/version, lint CLI, bind-address CLI, outbound HTTP budget, subprocess, + subprocess cleanup and subprocess security. +- `cargo fmt --all -- --check` and + `cargo clippy --all-targets --all-features -- -D warnings`: passed. +- Existing documentation validation: 36/36 passed. +- Static repository hygiene after staging all new files, and `git diff --check`: + passed. +- `cargo check --locked --manifest-path fuzz/Cargo.toml -j 1 --verbose`: + passed, including `fuzz_frontend`, in 5m22s. This is compilation evidence, + not a sustained fuzz campaign. + +The first default-parallel fuzz compilation failed while compiling unchanged +`crypto-common 0.2.2`: rustc exited 1 without an explaining compiler diagnostic. +The successful single-worker diagnostic build establishes that the candidate +fuzz workspace compiles, but does not identify the initial failure's cause. +Both logs are retained and the unresolved observation is tracked in +[issue #740](https://github.com/WebFirstLanguage/wfl/issues/740). No dependency, +lockfile or source change was made between those builds; the issue does not +waive any check. The host used stable x86_64-pc-windows-msvc Rust 1.98.1. + +Logs remain under ignored `target/reports/cli-budget/`. An independent reviewer +checked flag operands and placement, CLI modes and child argv, shared deadlines, +configured HTTP/operation/permission policies and cancellation cleanup. The +review's two fixture findings were fixed: the owned child now has its own longer +budget plus a positive control, and the child assertion program is launched in +test mode. Final source and fixture review identified no blocking finding. + +Exact-head CI must still complete, including both real five-minute boundary +steps and the ordinary Linux/Windows program sweeps. No merge or release is +authorized by this evidence record itself. diff --git a/History/dev-diary/2026/2026-09-20-cli-execution-budget.md b/History/dev-diary/2026/2026-09-20-cli-execution-budget.md new file mode 100644 index 00000000..44cc0442 --- /dev/null +++ b/History/dev-diary/2026/2026-09-20-cli-execution-budget.md @@ -0,0 +1,27 @@ +# A finite execution budget for long batch invocations + +The complete Scriptorium WFL suite exceeded the CLI's historical 300-second +cap on Linux. Increasing `.wflcfg` could not help because the cap was applied +after loading configuration. Making the test runner a lifetime-exempt server +loop would hide the boundary instead of supporting legitimate batch work. + +`wfl --execution-timeout 1200 batch.wfl` now grants one explicit finite deadline +to that invocation. Only `BudgetLimits.max_duration` changes; the interpreter +still receives its original capped configuration. This separation preserves +HTTP and streaming timeouts, main-loop process waits, explicit process-wait +deadlines, permissions and memory/operation/depth limits. Ordinary foreground +operations remain part of the same invocation budget, and executed files share +the same budget rather than resetting it. Child WFL processes retain their own +configuration unless the caller explicitly passes an override to them. + +The option accepts whole seconds from one through one year, rejects duplicate +or malformed operands, and must precede the source filename. Normal scripts and +test-mode scripts use the same option. Configuration maintenance, editor launch +and environment dumps refuse an irrelevant override instead of ignoring it. + +All new scenarios and drivers are WFL. Fast tests cover CLI routing, the existing +configuration baseline, longer/shorter deadlines, child cleanup and isolation, +and unchanged HTTP limits. A separate 305-second WFL case in both integration +jobs proves execution past the old ceiling with an explicit 330-second bound. +The existing outer CI bound remains finite. See the matching engineering +evidence record for Red/Green results, review and exact-head CI. diff --git a/README.md b/README.md index 6dd45f32..5845818e 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Other projects can run their WFL test scripts with the | `wfl --lint ` | Lint; add `--fix --in-place` to auto-fix | | `wfl --analyze ` | Static analysis | | `wfl --test ` | Run `describe`/`test` blocks | +| `wfl --execution-timeout 1200 ` | Give this invocation a finite 20-minute execution budget; preserve other limits | | `wfl --parse ` / `wfl --lex ` | Dump the AST / tokens | ## Documentation diff --git a/TestPrograms/cli_budget/arguments.test.wfl b/TestPrograms/cli_budget/arguments.test.wfl index e00482a8..9c7d99d2 100644 --- a/TestPrograms/cli_budget/arguments.test.wfl +++ b/TestPrograms/cli_budget/arguments.test.wfl @@ -60,4 +60,13 @@ describe "Explicit execution budget CLI arguments": expect outcome["error"] to contain "--execution-timeout must appear before the source filename" end for end test + + test "non-execution operations cannot silently ignore the budget": + store source_path as path_join of fixture_root and "arguments.wfl" + for each mode_option in ["--configCheck", "--configFix", "--edit", "--dump-env"]: + wait for execute command runtime_path with arguments ["--execution-timeout", "3", mode_option, source_path] as outcome + expect outcome["exit_code"] to equal 2 + expect outcome["error"] to contain "--execution-timeout requires source execution or analysis" + end for + end test end describe diff --git a/TestPrograms/cli_budget/deadlines.test.wfl b/TestPrograms/cli_budget/deadlines.test.wfl index a34a41cb..4296fc44 100644 --- a/TestPrograms/cli_budget/deadlines.test.wfl +++ b/TestPrograms/cli_budget/deadlines.test.wfl @@ -54,7 +54,7 @@ describe "Invocation-only execution deadlines": test "separate WFL children retain their own configuration deadline": store source_path as path_join of fixture_root and "child-config.wfl" - wait for execute command runtime_path with arguments ["--execution-timeout", "10", source_path] as outcome + wait for execute command runtime_path with arguments ["--execution-timeout", "10", "--test", source_path] as outcome expect outcome["success"] to equal yes expect outcome["output"] to contain "CHILD CONFIGURATION RETAINED" end test diff --git a/TestPrograms/cli_budget/resource-policy.test.wfl b/TestPrograms/cli_budget/resource-policy.test.wfl new file mode 100644 index 00000000..d5f1fb73 --- /dev/null +++ b/TestPrograms/cli_budget/resource-policy.test.wfl @@ -0,0 +1,32 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" + +describe "An execution duration override retains other ceilings": + test "a shorter explicit deadline overrides a longer configuration": + store source_path as path_join of fixture_root and "longer-config/wait.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path, "1500"] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "COMPLETED") to equal no + end test + + test "operation ceilings remain enforced during the invocation": + store source_path as path_join of fixture_root and "operations/loop.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", source_path] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "operation budget (100 operations)" + expect (outcome["output"] contains "MUST NOT ESCAPE") to equal no + end test + + test "an execution override cannot enable denied subprocesses": + store source_path as path_join of fixture_root and "denied/process.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", source_path] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "Command blocked by security policy" + expect (outcome["output"] contains "MUST NOT ESCAPE") to equal no + end test +end describe diff --git a/src/main.rs b/src/main.rs index 8219a251..5e1129be 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,6 +45,9 @@ fn print_help() { println!(" --output Specify an output file for the environment dump"); println!(" --time Measure and display execution time"); println!(" --test Run file in test mode"); + println!(" --execution-timeout "); + println!(" Set this invocation's finite execution budget (1–31536000)"); + println!(" Place before the source filename; other limits are unchanged"); println!(); println!("Configuration Maintenance:"); println!(" --configCheck Check configuration files for issues"); @@ -276,6 +279,7 @@ async fn run() -> io::Result<()> { let mut output_path = None; let mut time_mode = false; let mut test_mode = false; + let mut execution_timeout = None; let mut file_path = String::new(); let mut i = 1; @@ -292,6 +296,29 @@ async fn run() -> io::Result<()> { "--lint" | "--fix" | "--diff" | "--in-place" )); match args[i].as_str() { + "--execution-timeout" => { + if !file_path.is_empty() { + eprintln!("Error: --execution-timeout must appear before the source filename"); + process::exit(2); + } + if execution_timeout.is_some() { + eprintln!("Error: --execution-timeout may be specified only once"); + process::exit(2); + } + let seconds = args + .get(i + 1) + .filter(|value| !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit())) + .and_then(|value| value.parse::().ok()) + .filter(|seconds| (1..=31_536_000).contains(seconds)); + let Some(seconds) = seconds else { + eprintln!( + "Error: --execution-timeout requires a whole number of seconds from 1 to 31536000" + ); + process::exit(2); + }; + execution_timeout = Some(std::time::Duration::from_secs(seconds)); + i += 2; + } "--init" => { eprintln!("Error: initialization is a command. Use: wfl init"); process::exit(2); @@ -484,6 +511,14 @@ async fn run() -> io::Result<()> { // Validate the completed option set before running any operation or writing // output. Checking here makes conflicts independent of argument order. + if execution_timeout.is_some() + && (config_check_mode || config_fix_mode || edit_mode || dump_env_mode) + { + eprintln!( + "Error: --execution-timeout requires source execution or analysis; it cannot be combined with --configCheck, --configFix, --edit, or --dump-env" + ); + process::exit(2); + } if fix_diff && fix_in_place { eprintln!("Error: --in-place and --diff flags are mutually exclusive"); process::exit(2); @@ -622,14 +657,19 @@ async fn run() -> io::Result<()> { let script_dir = Path::new(&file_path).parent().unwrap_or(Path::new(".")); let config = config::load_config(script_dir); - // Build the ONE execution budget for this run up front, from the same - // (timeout-capped) config the interpreter will use, so a single budget - // governs the pre-parse source check, lexing/parsing/analysis, and - // interpretation — its deadline clock starts here and covers the whole run. + // Preserve the historic config timeout cap and its per-operation uses. + // An explicit invocation override changes ONLY the shared budget duration, + // not request/stream timeouts, main-loop subprocess waits, or other limits. + // The one deadline starts before reading/lexing/parsing/analysis and is + // reused by interpretation and nested executed files. let mut run_config = config.clone(); run_config.timeout_seconds = run_config.timeout_seconds.min(300); let run_config = std::sync::Arc::new(run_config); - let budget = std::sync::Arc::new(wfl::exec::budget::ExecutionBudget::from_config(&run_config)); + let mut budget_limits = wfl::exec::budget::BudgetLimits::from_config(&run_config); + if let Some(duration) = execution_timeout { + budget_limits.max_duration = Some(duration); + } + let budget = std::sync::Arc::new(wfl::exec::budget::ExecutionBudget::new(budget_limits)); // Install the run budget as the current-thread budget for the ENTIRE run, so // every front-end phase — lexing, parsing, analysis, type checking — and the @@ -1037,10 +1077,9 @@ async fn run() -> io::Result<()> { // Reuse the single budget (and the timeout-capped `run_config`) // built at the top of the run, so the source check and the // interpreter share one deadline/operation/cancellation budget. - // `run_config` carries the full `.wflcfg` (e.g. - // `web_server_bind_address`) with the 300s timeout cap applied - // (see issue #466); the cap never affects web servers because - // `check_time` skips the deadline inside a main loop. + // `run_config` preserves config policy, while `budget` may carry + // an explicit CLI execution duration. Main loops keep their + // existing lifetime exemption and finite operation timeouts. let mut interpreter = Interpreter::with_config_and_budget( std::sync::Arc::clone(&run_config), std::sync::Arc::clone(&budget), diff --git a/testing.md b/testing.md index 36c45a89..ecf61fa7 100644 --- a/testing.md +++ b/testing.md @@ -41,6 +41,7 @@ runs on Tokio. | Extension dependency security | `cd vscode-extension && npm ci && npm run test:security` | | Extension lint + VS Code host | `cargo build --locked -p wfl-lsp`, then `cd vscode-extension && npm ci && npm test` (use `xvfb-run -a npm test` on headless Linux) | | WFL end-to-end programs | `cargo build --release` then `./scripts/run_integration_tests.sh` (`.ps1` on Windows) | +| Long invocation boundary | `target/release/wfl --execution-timeout 330 --test tests/fixtures/cli_budget/long-run.test.wfl` (`wfl.exe` on Windows; about 305 seconds) | | Web-server end-to-end | `./scripts/run_web_tests.sh` (`.ps1` on Windows) | | Docs examples validation | `python scripts/validate_docs_examples.py` | | Benchmarks (perf, non-gating) | `cargo bench` | @@ -55,6 +56,7 @@ cargo fmt --all -- --check \ && (cd vscode-extension && npm ci && xvfb-run -a npm test) \ && cargo build --release \ && ./scripts/run_integration_tests.sh \ + && target/release/wfl --execution-timeout 330 --test tests/fixtures/cli_budget/long-run.test.wfl \ && ./scripts/run_web_tests.sh \ && python scripts/validate_docs_examples.py ``` @@ -126,6 +128,13 @@ Programs**, web tests, database tests, and fuzz-target compilation. All are required checks; the default branch MUST stay green. The nightly build is a release artifact and is separately monitored. +Both integration jobs also execute the WFL long-invocation boundary fixture +with a 330-second execution budget and a six-minute outer job-step limit. The +fixture asserts real ordinary execution beyond 300 seconds. The fast +`TestPrograms/cli_budget/` suites remain in the usual recursive program gates; +the long fixture lives under `tests/fixtures/` and is invoked explicitly rather +than skipped by the program sweep's 30-second limit. + The existing **Build, Test, Clippy** job also installs locked extension dependencies, runs the bounded dependency security regression, and runs the complete `npm test` command in a real VS Code host after building the LSP. diff --git a/tests/fixtures/cli_budget/denied/.wflcfg b/tests/fixtures/cli_budget/denied/.wflcfg new file mode 100644 index 00000000..914ead42 --- /dev/null +++ b/tests/fixtures/cli_budget/denied/.wflcfg @@ -0,0 +1,4 @@ +timeout_seconds = 1 +allow_shell_execution = false +execution_logging = false +debug_report_enabled = false diff --git a/tests/fixtures/cli_budget/denied/process.wfl b/tests/fixtures/cli_budget/denied/process.wfl new file mode 100644 index 00000000..9b8a5c8f --- /dev/null +++ b/tests/fixtures/cli_budget/denied/process.wfl @@ -0,0 +1,3 @@ +store runtime_path as call current_executable +wait for execute command runtime_path with arguments ["--version"] as outcome +display "MUST NOT ESCAPE PROCESS POLICY" diff --git a/tests/fixtures/cli_budget/longer-config/.wflcfg b/tests/fixtures/cli_budget/longer-config/.wflcfg new file mode 100644 index 00000000..edeb859c --- /dev/null +++ b/tests/fixtures/cli_budget/longer-config/.wflcfg @@ -0,0 +1,3 @@ +timeout_seconds = 10 +execution_logging = false +debug_report_enabled = false diff --git a/tests/fixtures/cli_budget/longer-config/wait.wfl b/tests/fixtures/cli_budget/longer-config/wait.wfl new file mode 100644 index 00000000..09a4243e --- /dev/null +++ b/tests/fixtures/cli_budget/longer-config/wait.wfl @@ -0,0 +1 @@ +include from "../wait.wfl" diff --git a/tests/fixtures/cli_budget/operations/.wflcfg b/tests/fixtures/cli_budget/operations/.wflcfg new file mode 100644 index 00000000..27f8a61a --- /dev/null +++ b/tests/fixtures/cli_budget/operations/.wflcfg @@ -0,0 +1,4 @@ +timeout_seconds = 1 +max_operations = 100 +execution_logging = false +debug_report_enabled = false diff --git a/tests/fixtures/cli_budget/operations/loop.wfl b/tests/fixtures/cli_budget/operations/loop.wfl new file mode 100644 index 00000000..750aa319 --- /dev/null +++ b/tests/fixtures/cli_budget/operations/loop.wfl @@ -0,0 +1,4 @@ +count from 1 to 5000: + store observed as count +end count +display "MUST NOT ESCAPE OPERATION LIMIT" diff --git a/tests/fixtures/cli_budget/shared-parent.wfl b/tests/fixtures/cli_budget/shared-parent.wfl index c0e22415..5853b19a 100644 --- a/tests/fixtures/cli_budget/shared-parent.wfl +++ b/tests/fixtures/cli_budget/shared-parent.wfl @@ -1,3 +1,3 @@ wait for 700 milliseconds -execute file "shared-child.wfl" +execute file at "shared-child.wfl" display "MUST NOT COMPLETE SHARED RUN" From 01416f4dcec1b3a33135c22a84b28230f350c1eb Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 07:01:41 -0500 Subject: [PATCH 3/9] test(cli): capture invocation deadline review regressions --- .../2026-09-20-cli-execution-budget.md | 13 ++++ .../cli_budget/review-boundaries.test.wfl | 69 +++++++++++++++++++ .../longer-config/server-client.wfl | 5 ++ tests/fixtures/cli_budget/main-loop-wait.wfl | 5 ++ .../cli_budget/wait-without-checkpoint.wfl | 2 + 5 files changed, 94 insertions(+) create mode 100644 TestPrograms/cli_budget/review-boundaries.test.wfl create mode 100644 tests/fixtures/cli_budget/longer-config/server-client.wfl create mode 100644 tests/fixtures/cli_budget/main-loop-wait.wfl create mode 100644 tests/fixtures/cli_budget/wait-without-checkpoint.wfl diff --git a/Engineering/evidence/2026-09-20-cli-execution-budget.md b/Engineering/evidence/2026-09-20-cli-execution-budget.md index 7b01049a..b882a3c1 100644 --- a/Engineering/evidence/2026-09-20-cli-execution-budget.md +++ b/Engineering/evidence/2026-09-20-cli-execution-budget.md @@ -110,3 +110,16 @@ test mode. Final source and fixture review identified no blocking finding. Exact-head CI must still complete, including both real five-minute boundary steps and the ordinary Linux/Windows program sweeps. No merge or release is authorized by this evidence record itself. + +## Review follow-up Red + +Automated review of `68c46650` exposed three additional semantic cases, verified +locally before their fixes in `TestPrograms/cli_budget/review-boundaries.test.wfl`. +The WFL suite passed its existing main-loop wait exemption baseline and failed +three real expectations: an ordinary five-second wait outlived a one-second +deadline, a one-second invocation override shortened a ten-second server HTTP +policy, and dump modes accepted a misplaced timeout after the source filename. +The log is `target/reports/cli-budget/red-review-boundaries.log` (1/4 passed). +The wait gap predates the CLI option; it matters to an explicit finite deadline +and cannot be hidden by adding a later operation checkpoint to the assertion. +These findings supersede the earlier source-review verdict until corrected. diff --git a/TestPrograms/cli_budget/review-boundaries.test.wfl b/TestPrograms/cli_budget/review-boundaries.test.wfl new file mode 100644 index 00000000..e2be9554 --- /dev/null +++ b/TestPrograms/cli_budget/review-boundaries.test.wfl @@ -0,0 +1,69 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" + +describe "Invocation budget review boundaries": + test "ordinary waits stop at the deadline without another checkpoint": + store source_path as path_join of fixture_root and "wait-without-checkpoint.wfl" + for each child_args in [[source_path], ["--execution-timeout", "1", source_path]]: + store started_at as current time in milliseconds + wait for execute command runtime_path with arguments child_args as outcome + store elapsed_ms as (current time in milliseconds) minus started_at + expect elapsed_ms is less than 3000 to equal yes + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT FINISH WAIT") to equal no + end for + end test + + test "server loop waits remain exempt from invocation lifetime": + store source_path as path_join of fixture_root and "main-loop-wait.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "SERVER WAIT RETAINED" + end test + + test "a shorter invocation budget does not shorten server HTTP policy": + store peer_path as path_join of fixture_root and "slow-peer.wfl" + wait for spawn command runtime_path with arguments [peer_path] as peer_process + try: + store startup_text as "" + store peer_url as "" + count from 1 to 200: + wait for read output from process peer_process as fresh_text + change startup_text to startup_text with fresh_text + for each output_line in (split startup_text by "\n"): + check if output_line starts with "READY ": + change peer_url to trim of (substring of output_line and 6 and ((length of output_line) minus 6)) + end check + end for + check if peer_url is not equal to "": + break + end check + wait for 10 milliseconds + end count + expect peer_url is not equal to "" to equal yes + store source_path as path_join of fixture_root and "longer-config/server-client.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path, peer_url] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "RECEIVED late response" + finally: + close process peer_process + end try + expect (process peer_process is running) to equal no + end test + + test "dump modes require the execution override before the filename": + store source_path as path_join of fixture_root and "arguments.wfl" + for each mode_option in ["--lex", "--ast", "--parse"]: + wait for execute command runtime_path with arguments [mode_option, source_path, "--execution-timeout", "3"] as misplaced + expect misplaced["exit_code"] to equal 2 + expect misplaced["error"] to contain "--execution-timeout must appear before the source filename" + wait for execute command runtime_path with arguments ["--execution-timeout", "3", mode_option, source_path] as correct + expect correct["success"] to equal yes + end for + end test +end describe diff --git a/tests/fixtures/cli_budget/longer-config/server-client.wfl b/tests/fixtures/cli_budget/longer-config/server-client.wfl new file mode 100644 index 00000000..1640bb6e --- /dev/null +++ b/tests/fixtures/cli_budget/longer-config/server-client.wfl @@ -0,0 +1,5 @@ +main loop: + open url at args[0] and read content as response_text + display "RECEIVED " with response_text + exit program +end loop diff --git a/tests/fixtures/cli_budget/main-loop-wait.wfl b/tests/fixtures/cli_budget/main-loop-wait.wfl new file mode 100644 index 00000000..4e61f90c --- /dev/null +++ b/tests/fixtures/cli_budget/main-loop-wait.wfl @@ -0,0 +1,5 @@ +main loop: + wait for 1500 milliseconds + display "SERVER WAIT RETAINED" + exit program +end loop diff --git a/tests/fixtures/cli_budget/wait-without-checkpoint.wfl b/tests/fixtures/cli_budget/wait-without-checkpoint.wfl new file mode 100644 index 00000000..9474572f --- /dev/null +++ b/tests/fixtures/cli_budget/wait-without-checkpoint.wfl @@ -0,0 +1,2 @@ +wait for 5000 milliseconds +display "MUST NOT FINISH WAIT" From 370ec22ad0dcf61b98e749464d3446453bf2f759 Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 07:03:10 -0500 Subject: [PATCH 4/9] test(cli): assert terminal waits cannot outlive execution budgets --- Engineering/evidence/2026-09-20-cli-execution-budget.md | 6 ++++++ TestPrograms/cli_budget/review-boundaries.test.wfl | 7 +++++++ tests/fixtures/cli_budget/wait-last.wfl | 1 + 3 files changed, 14 insertions(+) create mode 100644 tests/fixtures/cli_budget/wait-last.wfl diff --git a/Engineering/evidence/2026-09-20-cli-execution-budget.md b/Engineering/evidence/2026-09-20-cli-execution-budget.md index b882a3c1..77b69e9f 100644 --- a/Engineering/evidence/2026-09-20-cli-execution-budget.md +++ b/Engineering/evidence/2026-09-20-cli-execution-budget.md @@ -123,3 +123,9 @@ The log is `target/reports/cli-budget/red-review-boundaries.log` (1/4 passed). The wait gap predates the CLI option; it matters to an explicit finite deadline and cannot be hidden by adding a later operation checkpoint to the assertion. These findings supersede the earlier source-review verdict until corrected. +An additional last-statement wait case then confirmed successful exit after an +expired deadline without any following checkpoint; the expanded Red was 1/5 +(`red-review-boundaries-final-wait.log`). Existing embedded-runtime tests also +establish that a manually supplied 250ms budget bounds main-loop HTTP requests, +so the remedy must distinguish a CLI invocation override without changing that +existing API contract. diff --git a/TestPrograms/cli_budget/review-boundaries.test.wfl b/TestPrograms/cli_budget/review-boundaries.test.wfl index e2be9554..a966d236 100644 --- a/TestPrograms/cli_budget/review-boundaries.test.wfl +++ b/TestPrograms/cli_budget/review-boundaries.test.wfl @@ -19,6 +19,13 @@ describe "Invocation budget review boundaries": end for end test + test "a final duration wait still reports its expired deadline": + store source_path as path_join of fixture_root and "wait-last.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + end test + test "server loop waits remain exempt from invocation lifetime": store source_path as path_join of fixture_root and "main-loop-wait.wfl" wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path] as outcome diff --git a/tests/fixtures/cli_budget/wait-last.wfl b/tests/fixtures/cli_budget/wait-last.wfl new file mode 100644 index 00000000..f747568d --- /dev/null +++ b/tests/fixtures/cli_budget/wait-last.wfl @@ -0,0 +1 @@ +wait for 1500 milliseconds From a5988f1324e2417cbb25a68ba3340bc5ee8c894a Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 07:07:25 -0500 Subject: [PATCH 5/9] test(cli): preserve server stream limits under short invocation overrides --- .../cli_budget/stream-policy.test.wfl | 46 +++++++++++++++++++ .../longer-config/stream-client.wfl | 10 ++++ tests/fixtures/cli_budget/stream-peer.wfl | 22 +++++++++ 3 files changed, 78 insertions(+) create mode 100644 TestPrograms/cli_budget/stream-policy.test.wfl create mode 100644 tests/fixtures/cli_budget/longer-config/stream-client.wfl create mode 100644 tests/fixtures/cli_budget/stream-peer.wfl diff --git a/TestPrograms/cli_budget/stream-policy.test.wfl b/TestPrograms/cli_budget/stream-policy.test.wfl new file mode 100644 index 00000000..11b42184 --- /dev/null +++ b/TestPrograms/cli_budget/stream-policy.test.wfl @@ -0,0 +1,46 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" + +define action called assert_stream_policy with parameters request_path: + store peer_path as path_join of fixture_root and "stream-peer.wfl" + wait for spawn command runtime_path with arguments [peer_path] as peer_process + try: + store startup_text as "" + store peer_url as "" + count from 1 to 200: + wait for read output from process peer_process as fresh_text + change startup_text to startup_text with fresh_text + for each output_line in (split startup_text by "\n"): + check if output_line starts with "READY ": + change peer_url to trim of (substring of output_line and 6 and ((length of output_line) minus 6)) + end check + end for + check if peer_url is not equal to "": + break + end check + wait for 10 milliseconds + end count + expect peer_url is not equal to "" to equal yes + store source_path as path_join of fixture_root and "longer-config/stream-client.wfl" + store request_url as peer_url with request_path + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path, request_url] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "RECEIVED late stream response" + finally: + close process peer_process + end try + expect (process peer_process is running) to equal no +end action + +describe "Short invocation overrides retain server streaming policy": + test "delayed response headers use the configured server timeout": + call assert_stream_policy with "/head" + end test + test "delayed response body uses the configured stream timeout": + call assert_stream_policy with "/body" + end test +end describe diff --git a/tests/fixtures/cli_budget/longer-config/stream-client.wfl b/tests/fixtures/cli_budget/longer-config/stream-client.wfl new file mode 100644 index 00000000..23be2738 --- /dev/null +++ b/tests/fixtures/cli_budget/longer-config/stream-client.wfl @@ -0,0 +1,10 @@ +main loop: + open url at args[0] and stream response as upstream + try: + wait for next line from upstream as response_text + display "RECEIVED " with response_text + finally: + close upstream + end try + exit program +end loop diff --git a/tests/fixtures/cli_budget/stream-peer.wfl b/tests/fixtures/cli_budget/stream-peer.wfl new file mode 100644 index 00000000..a3a28cf2 --- /dev/null +++ b/tests/fixtures/cli_budget/stream-peer.wfl @@ -0,0 +1,22 @@ +listen on port 0 as stream_peer +store listener_text as "" with stream_peer +display "READY http://" with (substring of listener_text and 11 and ((length of listener_text) minus 11)) +main loop: + wait for request comes in on stream_peer as incoming + check if (path of incoming) is equal to "/head": + wait for 2500 milliseconds + end check + try: + start streaming response to incoming with status 200 and content type "text/plain" as outgoing + try: + check if (path of incoming) is equal to "/body": + wait for 2500 milliseconds + end check + write line "late stream response" to outgoing + finally: + close outgoing + end try + when error: + display "TIMED CLIENT DISCONNECTED" + end try +end loop From 028ef59ed34aa0a515813010787a2eedec01f54f Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 07:13:12 -0500 Subject: [PATCH 6/9] test(cli): capture duration wait resource cleanup regressions --- .../cli_budget/wait-resources.test.wfl | 138 ++++++++++++++++++ .../cli_budget/wait-resources/.wflcfg | 7 + .../wait-resources/delayed-writer.wfl | 7 + .../wait-resources/owned-duration-wait.wfl | 21 +++ .../websocket-duration-wait.wfl | 20 +++ 5 files changed, 193 insertions(+) create mode 100644 TestPrograms/cli_budget/wait-resources.test.wfl create mode 100644 tests/fixtures/cli_budget/wait-resources/.wflcfg create mode 100644 tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl create mode 100644 tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl create mode 100644 tests/fixtures/cli_budget/wait-resources/websocket-duration-wait.wfl diff --git a/TestPrograms/cli_budget/wait-resources.test.wfl b/TestPrograms/cli_budget/wait-resources.test.wfl new file mode 100644 index 00000000..5501f664 --- /dev/null +++ b/TestPrograms/cli_budget/wait-resources.test.wfl @@ -0,0 +1,138 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget/wait-resources" +store case_root as path_join of repo_root and "target/test-artifacts/cli-budget-wait-resources" and (generate_uuid) +call makedirs with case_root + +define action called resource_read_text with parameters source_path: + open file at source_path for reading as input_file + try: + wait for store text_value as read content from input_file + return text_value + finally: + close file input_file + end try +end action + +define action called resource_wait_ready with parameters child_process and ready_path: + count from 1 to 200: + check if is_file of ready_path: + return yes + end check + check if (process child_process is running) is equal to no: + break + end check + wait for 10 milliseconds + end count + return no +end action + +define action called resource_rebind_port with parameters listener_port: + listen for websockets on port listener_port as rebound_server + close server rebound_server + return yes +end action + +describe "Duration waits release owned resources at their deadline": + teardown: + call remove_dir with case_root and yes + end teardown + + test "a duration wait timeout stops its ready child before a delayed write": + store writer_path as path_join of fixture_root and "delayed-writer.wfl" + store control_ready as path_join of case_root and "control.ready" + store control_marker as path_join of case_root and "control.late" + // Positive control: the same writer really creates its delayed file. + wait for execute command runtime_path with arguments ["--execution-timeout", "8", writer_path, control_ready, control_marker] as control_outcome + expect control_outcome["success"] to equal yes + expect is_file of control_ready to equal yes + expect resource_read_text of control_marker to equal "delayed writer survived" + + store ready_path as path_join of case_root and "owned.ready" + store marker_path as path_join of case_root and "owned.late" + store parent_path as path_join of fixture_root and "owned-duration-wait.wfl" + store started_at as current time in milliseconds + store outcome as nothing + wait for spawn command runtime_path with arguments ["--execution-timeout", "1", parent_path, ready_path, marker_path] as parent_process + try: + wait for process parent_process to complete with timeout 7 and read result as completed_parent + change outcome to completed_parent + finally: + close process parent_process + end try + store elapsed_ms as (current time in milliseconds) minus started_at + // Observe past the child's normal two-second write time, even when the + // owner correctly stopped at one second. No test-side kill supplies it. + check if elapsed_ms is less than 3000: + wait for (3000 minus elapsed_ms) milliseconds + end check + display "Duration wait owner: elapsed=" with elapsed_ms with ", success=" with outcome["success"] with ", late_write=" with (is_file of marker_path) + expect is_file of ready_path to equal yes + expect outcome["output"] to contain "READY CHILD BEFORE DURATION WAIT" + expect elapsed_ms is less than 3000 to equal yes + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT FINISH OWNED DURATION WAIT") to equal no + expect is_file of marker_path to equal no + end test + + test "an active WebSocket receiver times out and releases its exact port": + store ready_path as path_join of case_root and "websocket-timeout.ready" + store source_path as path_join of fixture_root and "websocket-duration-wait.wfl" + store started_at as current time in milliseconds + store outcome as nothing + store listener_port as 0 + wait for spawn command runtime_path with arguments ["--execution-timeout", "1", source_path, ready_path, "ordinary"] as server_process + try: + expect resource_wait_ready of server_process and ready_path to equal yes + change listener_port to parse_json of (resource_read_text of ready_path) + expect listener_port is greater than 0 to equal yes + // Prove this is the child's live port, not an unrelated free port. + store refused_while_live as no + try: + store unexpected_bind as resource_rebind_port of listener_port + when error: + change refused_while_live to yes + expect error_message to contain "Failed to start websocket server" + end try + expect refused_while_live to equal yes + wait for process server_process to complete with timeout 7 and read result as completed_server + change outcome to completed_server + finally: + close process server_process + end try + store elapsed_ms as (current time in milliseconds) minus started_at + store rebound as resource_rebind_port of listener_port + display "WebSocket duration wait: elapsed=" with elapsed_ms with ", success=" with outcome["success"] with ", rebound=" with rebound + expect elapsed_ms is less than 3000 to equal yes + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT FINISH WEBSOCKET WAIT") to equal no + expect rebound to equal yes + end test + + test "a WebSocket receiver in a main loop keeps its exemption and releases its port": + store ready_path as path_join of case_root and "websocket-loop.ready" + store source_path as path_join of fixture_root and "websocket-duration-wait.wfl" + store started_at as current time in milliseconds + store outcome as nothing + store listener_port as 0 + wait for spawn command runtime_path with arguments ["--execution-timeout", "1", source_path, ready_path, "main-loop"] as server_process + try: + expect resource_wait_ready of server_process and ready_path to equal yes + change listener_port to parse_json of (resource_read_text of ready_path) + wait for process server_process to complete with timeout 5 and read result as completed_server + change outcome to completed_server + finally: + close process server_process + end try + store elapsed_ms as (current time in milliseconds) minus started_at + expect outcome["success"] to equal yes + expect outcome["output"] to contain "EXEMPT WEBSOCKET WAIT FINISHED" + expect elapsed_ms is greater than or equal to 1400 to equal yes + expect resource_rebind_port of listener_port to equal yes + end test +end describe diff --git a/tests/fixtures/cli_budget/wait-resources/.wflcfg b/tests/fixtures/cli_budget/wait-resources/.wflcfg new file mode 100644 index 00000000..8a7a852c --- /dev/null +++ b/tests/fixtures/cli_budget/wait-resources/.wflcfg @@ -0,0 +1,7 @@ +timeout_seconds = 10 +execution_logging = false +debug_report_enabled = false +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true +web_server_bind_address = 127.0.0.1 diff --git a/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl b/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl new file mode 100644 index 00000000..211aeac8 --- /dev/null +++ b/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl @@ -0,0 +1,7 @@ +open file at args[0] for writing as ready_file +wait for write content "ready" into ready_file +close file ready_file +wait for 2000 milliseconds +open file at args[1] for writing as marker_file +wait for write content "delayed writer survived" into marker_file +close file marker_file diff --git a/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl b/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl new file mode 100644 index 00000000..d3220ced --- /dev/null +++ b/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl @@ -0,0 +1,21 @@ +store runtime_path as call current_executable +store writer_path as path_join of script_directory and "delayed-writer.wfl" +wait for spawn command runtime_path with arguments ["--execution-timeout", "8", writer_path, args[0], args[1]] as owned_child +try: + store ready_seen as no + count from 1 to 100: + check if is_file of args[0]: + change ready_seen to yes + break + end check + wait for 10 milliseconds + end count + check if ready_seen is equal to no: + call raise_error with "Delayed writer did not signal readiness" + end check + display "READY CHILD BEFORE DURATION WAIT" + wait for 4500 milliseconds + display "MUST NOT FINISH OWNED DURATION WAIT" +finally: + close process owned_child +end try diff --git a/tests/fixtures/cli_budget/wait-resources/websocket-duration-wait.wfl b/tests/fixtures/cli_budget/wait-resources/websocket-duration-wait.wfl new file mode 100644 index 00000000..469320b8 --- /dev/null +++ b/tests/fixtures/cli_budget/wait-resources/websocket-duration-wait.wfl @@ -0,0 +1,20 @@ +listen for websockets on port 0 as waiting_server +store address_parts as split waiting_server by ":" +store listener_port as address_parts[(length of address_parts) minus 1] +open file at args[0] for writing as ready_file +wait for write content listener_port into ready_file +close file ready_file +try: + check if args[1] is equal to "main-loop": + main loop: + wait for 1500 milliseconds + display "EXEMPT WEBSOCKET WAIT FINISHED" + exit program + end loop + otherwise: + wait for 4500 milliseconds + display "MUST NOT FINISH WEBSOCKET WAIT" + end check +finally: + close server waiting_server +end try From 5d781336e0a04bc66d43fc137727d23b4c68e48b Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 07:19:10 -0500 Subject: [PATCH 7/9] test(cli): observe leaked wait resources before driver cleanup --- .../cli_budget/wait-resources.test.wfl | 102 +++++++++--------- .../wait-resources/delayed-writer.wfl | 12 ++- .../wait-resources/owned-duration-wait.wfl | 4 +- 3 files changed, 65 insertions(+), 53 deletions(-) diff --git a/TestPrograms/cli_budget/wait-resources.test.wfl b/TestPrograms/cli_budget/wait-resources.test.wfl index 5501f664..80728f5c 100644 --- a/TestPrograms/cli_budget/wait-resources.test.wfl +++ b/TestPrograms/cli_budget/wait-resources.test.wfl @@ -17,14 +17,11 @@ define action called resource_read_text with parameters source_path: end try end action -define action called resource_wait_ready with parameters child_process and ready_path: +define action called resource_wait_ready with parameters ready_path: count from 1 to 200: check if is_file of ready_path: return yes end check - check if (process child_process is running) is equal to no: - break - end check wait for 10 milliseconds end count return no @@ -36,6 +33,15 @@ define action called resource_rebind_port with parameters listener_port: return yes end action +define action called resource_release_writer with parameters release_path: + open file at release_path for writing as release_file + try: + wait for write content "release" into release_file + finally: + close file release_file + end try +end action + describe "Duration waits release owned resources at their deadline": teardown: call remove_dir with case_root and yes @@ -45,94 +51,90 @@ describe "Duration waits release owned resources at their deadline": store writer_path as path_join of fixture_root and "delayed-writer.wfl" store control_ready as path_join of case_root and "control.ready" store control_marker as path_join of case_root and "control.late" - // Positive control: the same writer really creates its delayed file. - wait for execute command runtime_path with arguments ["--execution-timeout", "8", writer_path, control_ready, control_marker] as control_outcome + store control_release as path_join of case_root and "control.release" + // Positive control: the same writer creates its file after release. + call resource_release_writer with control_release + wait for execute command runtime_path with arguments ["--execution-timeout", "10", writer_path, control_ready, control_marker, control_release] as control_outcome expect control_outcome["success"] to equal yes expect is_file of control_ready to equal yes expect resource_read_text of control_marker to equal "delayed writer survived" store ready_path as path_join of case_root and "owned.ready" store marker_path as path_join of case_root and "owned.late" + store release_path as path_join of case_root and "owned.release" store parent_path as path_join of fixture_root and "owned-duration-wait.wfl" store started_at as current time in milliseconds - store outcome as nothing - wait for spawn command runtime_path with arguments ["--execution-timeout", "1", parent_path, ready_path, marker_path] as parent_process + wait for spawn command runtime_path with arguments ["--execution-timeout", "1", parent_path, ready_path, marker_path, release_path] as parent_process try: - wait for process parent_process to complete with timeout 7 and read result as completed_parent - change outcome to completed_parent + expect resource_wait_ready of ready_path to equal yes + // Neither poll nor reap the owner before this observation: native + // process completion also closes its tree and could hide a leak. + // Release the ready writer after its owner's deadline. The old + // unbounded duration wait is still active throughout this window. + wait for 1500 milliseconds + call resource_release_writer with release_path + store late_before_reap as resource_wait_ready of marker_path + wait for process parent_process to complete with timeout 7 and read result as outcome + store elapsed_ms as (current time in milliseconds) minus started_at + display "Duration wait owner: elapsed=" with elapsed_ms with ", success=" with outcome["success"] with ", late_write_before_reap=" with late_before_reap + expect outcome["output"] to contain "READY CHILD BEFORE DURATION WAIT" + expect late_before_reap to equal no + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT FINISH OWNED DURATION WAIT") to equal no finally: close process parent_process end try - store elapsed_ms as (current time in milliseconds) minus started_at - // Observe past the child's normal two-second write time, even when the - // owner correctly stopped at one second. No test-side kill supplies it. - check if elapsed_ms is less than 3000: - wait for (3000 minus elapsed_ms) milliseconds - end check - display "Duration wait owner: elapsed=" with elapsed_ms with ", success=" with outcome["success"] with ", late_write=" with (is_file of marker_path) - expect is_file of ready_path to equal yes - expect outcome["output"] to contain "READY CHILD BEFORE DURATION WAIT" - expect elapsed_ms is less than 3000 to equal yes - expect outcome["success"] to equal no - expect outcome["error"] to contain "timeout (1s)" - expect (outcome["output"] contains "MUST NOT FINISH OWNED DURATION WAIT") to equal no - expect is_file of marker_path to equal no end test test "an active WebSocket receiver times out and releases its exact port": store ready_path as path_join of case_root and "websocket-timeout.ready" store source_path as path_join of fixture_root and "websocket-duration-wait.wfl" store started_at as current time in milliseconds - store outcome as nothing - store listener_port as 0 wait for spawn command runtime_path with arguments ["--execution-timeout", "1", source_path, ready_path, "ordinary"] as server_process try: - expect resource_wait_ready of server_process and ready_path to equal yes - change listener_port to parse_json of (resource_read_text of ready_path) + expect resource_wait_ready of ready_path to equal yes + store listener_port as parse_json of (resource_read_text of ready_path) expect listener_port is greater than 0 to equal yes // Prove this is the child's live port, not an unrelated free port. store refused_while_live as no try: - store unexpected_bind as resource_rebind_port of listener_port + call resource_rebind_port with listener_port when error: change refused_while_live to yes expect error_message to contain "Failed to start websocket server" end try expect refused_while_live to equal yes - wait for process server_process to complete with timeout 7 and read result as completed_server - change outcome to completed_server + wait for process server_process to complete with timeout 7 and read result as outcome + store elapsed_ms as (current time in milliseconds) minus started_at + store rebound as resource_rebind_port of listener_port + display "WebSocket duration wait: elapsed=" with elapsed_ms with ", success=" with outcome["success"] with ", rebound=" with rebound + expect elapsed_ms is less than 3000 to equal yes + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT FINISH WEBSOCKET WAIT") to equal no + expect rebound to equal yes finally: close process server_process end try - store elapsed_ms as (current time in milliseconds) minus started_at - store rebound as resource_rebind_port of listener_port - display "WebSocket duration wait: elapsed=" with elapsed_ms with ", success=" with outcome["success"] with ", rebound=" with rebound - expect elapsed_ms is less than 3000 to equal yes - expect outcome["success"] to equal no - expect outcome["error"] to contain "timeout (1s)" - expect (outcome["output"] contains "MUST NOT FINISH WEBSOCKET WAIT") to equal no - expect rebound to equal yes end test test "a WebSocket receiver in a main loop keeps its exemption and releases its port": store ready_path as path_join of case_root and "websocket-loop.ready" store source_path as path_join of fixture_root and "websocket-duration-wait.wfl" store started_at as current time in milliseconds - store outcome as nothing - store listener_port as 0 wait for spawn command runtime_path with arguments ["--execution-timeout", "1", source_path, ready_path, "main-loop"] as server_process try: - expect resource_wait_ready of server_process and ready_path to equal yes - change listener_port to parse_json of (resource_read_text of ready_path) - wait for process server_process to complete with timeout 5 and read result as completed_server - change outcome to completed_server + expect resource_wait_ready of ready_path to equal yes + store listener_port as parse_json of (resource_read_text of ready_path) + wait for process server_process to complete with timeout 5 and read result as outcome + store elapsed_ms as (current time in milliseconds) minus started_at + expect outcome["success"] to equal yes + expect outcome["output"] to contain "EXEMPT WEBSOCKET WAIT FINISHED" + expect elapsed_ms is greater than or equal to 1400 to equal yes + expect resource_rebind_port of listener_port to equal yes finally: close process server_process end try - store elapsed_ms as (current time in milliseconds) minus started_at - expect outcome["success"] to equal yes - expect outcome["output"] to contain "EXEMPT WEBSOCKET WAIT FINISHED" - expect elapsed_ms is greater than or equal to 1400 to equal yes - expect resource_rebind_port of listener_port to equal yes end test end describe diff --git a/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl b/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl index 211aeac8..04fb5c98 100644 --- a/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl +++ b/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl @@ -1,7 +1,17 @@ open file at args[0] for writing as ready_file wait for write content "ready" into ready_file close file ready_file -wait for 2000 milliseconds +store release_seen as no +count from 1 to 160: + check if is_file of args[2]: + change release_seen to yes + break + end check + wait for 50 milliseconds +end count +check if release_seen is equal to no: + call raise_error with "Delayed writer was never released" +end check open file at args[1] for writing as marker_file wait for write content "delayed writer survived" into marker_file close file marker_file diff --git a/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl b/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl index d3220ced..b03a515a 100644 --- a/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl +++ b/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl @@ -1,6 +1,6 @@ store runtime_path as call current_executable store writer_path as path_join of script_directory and "delayed-writer.wfl" -wait for spawn command runtime_path with arguments ["--execution-timeout", "8", writer_path, args[0], args[1]] as owned_child +wait for spawn command runtime_path with arguments ["--execution-timeout", "10", writer_path, args[0], args[1], args[2]] as owned_child try: store ready_seen as no count from 1 to 100: @@ -14,7 +14,7 @@ try: call raise_error with "Delayed writer did not signal readiness" end check display "READY CHILD BEFORE DURATION WAIT" - wait for 4500 milliseconds + wait for 6500 milliseconds display "MUST NOT FINISH OWNED DURATION WAIT" finally: close process owned_child From 30ed9462c8be758d3ab293c67e197fd6df9a967d Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 07:21:33 -0500 Subject: [PATCH 8/9] fix(cli): enforce waits and preserve server operation budgets --- Docs/reference/configuration-reference.md | 3 + .../2026-09-20-cli-execution-budget.md | 56 +++++++++++++++++++ .../2026/2026-09-20-cli-execution-budget.md | 11 ++++ src/exec/budget.rs | 18 ++++++ src/interpreter/mod.rs | 38 ++++++++++--- src/main.rs | 18 ++++-- 6 files changed, 131 insertions(+), 13 deletions(-) diff --git a/Docs/reference/configuration-reference.md b/Docs/reference/configuration-reference.md index 96e0b921..96d5075e 100644 --- a/Docs/reference/configuration-reference.md +++ b/Docs/reference/configuration-reference.md @@ -313,6 +313,9 @@ executed files. It does not reset between tests or nested files, and it does not implicitly change the configuration of separately launched WFL children. Ordinary foreground subprocess operations continue sharing the invocation deadline, while explicit process-wait timeouts remain independently enforced. +Duration waits observe cancellation and the invocation deadline while waiting, +including a final wait with no following statement. Waits within an active +`main loop` retain the server lifetime exemption. The override does not change other resource limits, subprocess permissions, request/stream limits, or configured per-operation timeouts. A server's `main diff --git a/Engineering/evidence/2026-09-20-cli-execution-budget.md b/Engineering/evidence/2026-09-20-cli-execution-budget.md index 77b69e9f..3e26d9d1 100644 --- a/Engineering/evidence/2026-09-20-cli-execution-budget.md +++ b/Engineering/evidence/2026-09-20-cli-execution-budget.md @@ -129,3 +129,59 @@ expired deadline without any following checkpoint; the expanded Red was 1/5 establish that a manually supplied 250ms budget bounds main-loop HTTP requests, so the remedy must distinguish a CLI invocation override without changing that existing API contract. + +Additional WFL streamed-response tests failed 0/2 on the frozen `68c46650` +binary, covering delayed headers and delayed body reads under config 10s / CLI +1s. Their first draft used a reserved parameter name and an ungrouped list +expression; those fixture parse errors were corrected before the recorded +semantic Red. An independently authored resource suite also failed its ordinary +wait and active WebSocket wait cases while passing the main-loop exemption +baseline (1/3). Red commits are `01416f4d`, `370ec22a`, `a5988f13`, and `028ef59e`. + +The final remedy retains the original main-loop operation duration privately in +`ExecutionBudget`. Only `with_invocation_timeout` replaces run lifetime; existing +explicit-budget constructors retain their behavior. HTTP continues choosing the +minimum of that operation duration and the configured/remaining stream limit. +Duration waits now check eagerly before/after the wait and each WebSocket pump +iteration. Passive sleeping and receiving use at most 10ms intervals; handler +dispatch is awaited normally so WFL finally blocks and interpreter state unwind. +A reviewed intermediate outer-select design was rejected because dropping an +arbitrary running handler could skip that cleanup. Its successful checks are +not final-source acceptance evidence. Dump modes scan only for a misplaced new +option, preserving the handling of unrelated trailing arguments. + +The resource regression was strengthened in `5d781336` after review found that +driver-side process reaping/closing could mask a surviving descendant. The +fixture now establishes writer readiness, releases it after the owner's +deadline, and observes the marker before any parent poll/reap. Exact-port +WebSocket rebind also happens before the driver's cleanup. The final frozen +old-binary result is 1/3: the late-write assertion fails with an actual pre-reap +write, the ordinary WebSocket wait outlives the deadline, and the exempt server +case passes. The unchanged final-source suite passes 3/3 in about eight seconds. +This covers an idle active WebSocket receiver; queued handler traffic remains +covered by the existing Rust WebSocket suite, not by this new WFL fixture. + +The final reviewed no-drop candidate SHA-256 is +`c0619c544b09551ce7988f58a564f50ff04e48a5e94a6f903c08a141b911c516`. +At this source, all seven fast WFL suites pass 25/25. `cargo test --all --locked` +passes 2,414 tests, with 27 existing ignored tests across 175 result records; +this includes the unchanged custom-250ms main-loop HTTP contract, stream tests, +WebSocket tests and CLI compatibility. Release build, fmt, strict Clippy, +fuzz-target compilation, and documentation validation (36/36) pass. Local logs +are the `*-final*` and `wait-resources-green.log` files under the same ignored +report directory. Final source review found no remaining blocking issue. + +The first remote run for `68c46650`, +[CI 35509162373](https://github.com/WebFirstLanguage/wfl/actions/runs/35509162373), +is not acceptance evidence: Windows integration failed in unchanged +`trusted_proxy_test::default_and_untrusted_peers_ignore_forged_forwarding` when +binding port 53684 reported Windows address-in-use error 10048. Its Linux +integration sibling was then cancelled, so neither long-boundary step ran. +Both ordinary program sweeps and the other completed jobs passed. The fixture +obtains a free port before spawning the server, which leaves a reuse window; +the observed log does not establish who occupied it. The final local full Rust +run passes that test unchanged. A new exact-head CI run must pass all gates; +the earlier failure is retained rather than treated as a waiver. + +The final no-drop candidate also passes the actual 305-second WFL boundary +1/1 with `--execution-timeout 330`; `green-final-long.log` records the result. diff --git a/History/dev-diary/2026/2026-09-20-cli-execution-budget.md b/History/dev-diary/2026/2026-09-20-cli-execution-budget.md index 44cc0442..c43f9f79 100644 --- a/History/dev-diary/2026/2026-09-20-cli-execution-budget.md +++ b/History/dev-diary/2026/2026-09-20-cli-execution-budget.md @@ -25,3 +25,14 @@ and unchanged HTTP limits. A separate 305-second WFL case in both integration jobs proves execution past the old ceiling with an explicit 330-second bound. The existing outer CI bound remains finite. See the matching engineering evidence record for Red/Green results, review and exact-head CI. + +Review exposed two policy boundaries that needed stronger tests: a shorter +invocation override must not shorten main-loop HTTP or streamed responses, and +a duration wait must observe the deadline even when it is the final statement. +The budget now retains its original operation duration separately when the CLI +overrides invocation lifetime, preserving existing embedded custom budgets. +Duration waits check eagerly and poll passive sleeping/receiving in bounded +intervals. WebSocket handlers are awaited normally so errors run their cleanup +and unwind interpreter state; the runtime does not cancel a handler future to +interrupt a duration wait. Dump modes reject the misplaced new option while +preserving their handling of unrelated trailing arguments. diff --git a/src/exec/budget.rs b/src/exec/budget.rs index d2d288b0..aa7d0322 100644 --- a/src/exec/budget.rs +++ b/src/exec/budget.rs @@ -340,6 +340,9 @@ impl std::error::Error for BudgetExceeded {} #[derive(Debug)] pub struct ExecutionBudget { limits: BudgetLimits, + /// Original finite operation ceiling for deadline-exempt main-loop HTTP. + /// An explicit CLI invocation override changes the run lifetime only. + main_loop_duration: Option, started: Instant, cancelled: AtomicBool, /// Total interpreter operations charged. Also drives the clock-sampling @@ -377,6 +380,7 @@ impl ExecutionBudget { /// Build a budget from explicit limits, starting the deadline clock now. pub fn new(limits: BudgetLimits) -> Self { Self { + main_loop_duration: limits.max_duration, limits, started: Instant::now(), cancelled: AtomicBool::new(false), @@ -388,6 +392,20 @@ impl ExecutionBudget { } } + /// Override only the invocation lifetime, preserving the original finite + /// main-loop operation ceiling. Existing explicit-budget constructors keep + /// their historical shared lifetime/operation-duration behavior. + pub fn with_invocation_timeout(limits: BudgetLimits, duration: Duration) -> Self { + let mut budget = Self::new(limits); + budget.limits.max_duration = Some(duration); + budget + } + + /// Original operation duration, unaffected by a CLI lifetime override. + pub fn main_loop_duration(&self) -> Option { + self.main_loop_duration + } + /// Build a budget from a loaded configuration. See /// [`BudgetLimits::from_config`]. pub fn from_config(config: &WflConfig) -> Self { diff --git a/src/interpreter/mod.rs b/src/interpreter/mod.rs index cb86f7c1..f60068cf 100644 --- a/src/interpreter/mod.rs +++ b/src/interpreter/mod.rs @@ -3899,7 +3899,7 @@ impl IoClient { if budget.is_deadline_exempt() { return Ok(OutboundHttpDeadline::MainLoop { - duration: budget.limits().max_duration.unwrap_or(configured_timeout), + duration: budget.main_loop_duration().unwrap_or(configured_timeout), }); } @@ -9605,10 +9605,14 @@ impl Interpreter { // While WebSocket servers are running, spend the wait window // dispatching their events to the registered handler blocks. - // With no WebSocket servers this is an ordinary sleep, so the - // statement's timing semantics are unchanged. - self.pump_websocket_events(std::time::Duration::from_millis(duration_ms)) + // The pump checks eagerly and bounds its sleeping/receiving + // intervals, while awaiting WFL handlers normally so their + // finally blocks and interpreter frames always unwind. + self.check_wait_budget(*line, *column)?; + self.pump_websocket_events(Duration::from_millis(duration_ms)) .await?; + // A final wait must fail even without a later sampled check. + self.check_wait_budget(*line, *column)?; Ok((Value::Null, ControlFlow::None)) } Statement::TryStatement { @@ -13921,17 +13925,35 @@ impl Interpreter { )) } + /// Eager deadline/cancellation check for asynchronous duration waits. + fn check_wait_budget(&self, line: usize, column: usize) -> Result<(), RuntimeError> { + self.budget + .check_cancelled() + .map_err(|e| self.budget_error(e, line, column))?; + if !self.budget.is_deadline_exempt() { + self.budget + .check_deadline() + .map_err(|e| self.budget_error(e, line, column))?; + } + Ok(()) + } + /// Drains and dispatches queued websocket events for up to `budget`, running /// the matching handler block for each. With no websocket servers active it /// is a plain sleep, preserving `wait for ` semantics. async fn pump_websocket_events(&self, budget: Duration) -> Result<(), RuntimeError> { let deadline = tokio::time::Instant::now() + budget; loop { + // Check even when queued events or closed channels remain ready. + self.check_wait_budget(0, 0)?; let now = tokio::time::Instant::now(); if now >= deadline { break; } let remaining = deadline - now; + // Poll only the passive wait, not an arbitrary WFL handler future: + // dropping a handler could skip finally and leave action state. + let wait_slice = remaining.min(Duration::from_millis(10)); // Snapshot the receivers with a short borrow; dispatch below must not // hold a borrow of web_socket_servers across handler execution. @@ -13943,8 +13965,8 @@ impl Interpreter { .collect(); if receivers.is_empty() { - tokio::time::sleep(remaining).await; - break; + tokio::time::sleep(wait_slice).await; + continue; } let mut recv_futs = Vec::with_capacity(receivers.len()); @@ -13956,11 +13978,11 @@ impl Interpreter { })); } - let sleep_fut = tokio::time::sleep(remaining); + let sleep_fut = tokio::time::sleep(wait_slice); tokio::pin!(sleep_fut); tokio::select! { - _ = &mut sleep_fut => break, + _ = &mut sleep_fut => {}, ((key, event), _idx, _rest) = futures_util::future::select_all(recv_futs) => { if let Some(event) = event { self.dispatch_ws_event(&key, event).await?; diff --git a/src/main.rs b/src/main.rs index 5e1129be..3827be59 100644 --- a/src/main.rs +++ b/src/main.rs @@ -511,6 +511,12 @@ async fn run() -> io::Result<()> { // Validate the completed option set before running any operation or writing // output. Checking here makes conflicts independent of argument order. + // Dump modes do not execute script arguments, but historically ignore them. + // Reject only this misplaced new option without reinterpreting legacy argv. + if (lex_dump || ast_dump) && args[i..].iter().any(|arg| arg == "--execution-timeout") { + eprintln!("Error: --execution-timeout must appear before the source filename"); + process::exit(2); + } if execution_timeout.is_some() && (config_check_mode || config_fix_mode || edit_mode || dump_env_mode) { @@ -665,11 +671,13 @@ async fn run() -> io::Result<()> { let mut run_config = config.clone(); run_config.timeout_seconds = run_config.timeout_seconds.min(300); let run_config = std::sync::Arc::new(run_config); - let mut budget_limits = wfl::exec::budget::BudgetLimits::from_config(&run_config); - if let Some(duration) = execution_timeout { - budget_limits.max_duration = Some(duration); - } - let budget = std::sync::Arc::new(wfl::exec::budget::ExecutionBudget::new(budget_limits)); + let budget_limits = wfl::exec::budget::BudgetLimits::from_config(&run_config); + let budget = std::sync::Arc::new(match execution_timeout { + Some(duration) => { + wfl::exec::budget::ExecutionBudget::with_invocation_timeout(budget_limits, duration) + } + None => wfl::exec::budget::ExecutionBudget::new(budget_limits), + }); // Install the run budget as the current-thread budget for the ENTIRE run, so // every front-end phase — lexing, parsing, analysis, type checking — and the From 358dc9eb15f61152f75f816d9e51c396cd24dd77 Mon Sep 17 00:00:00 2001 From: Brad Byrd Date: Sun, 20 Sep 2026 07:24:57 -0500 Subject: [PATCH 9/9] test: bind trusted proxy fixtures to owned ephemeral ports --- .../2026-09-20-cli-execution-budget.md | 20 ++++++++++++++ tests/trusted_proxy_test.rs | 27 ++++++++++++++----- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/Engineering/evidence/2026-09-20-cli-execution-budget.md b/Engineering/evidence/2026-09-20-cli-execution-budget.md index 3e26d9d1..ccaa2868 100644 --- a/Engineering/evidence/2026-09-20-cli-execution-budget.md +++ b/Engineering/evidence/2026-09-20-cli-execution-budget.md @@ -185,3 +185,23 @@ the earlier failure is retained rather than treated as a waiver. The final no-drop candidate also passes the actual 305-second WFL boundary 1/1 with `--execution-timeout 330`; `green-final-long.log` records the result. + +The final candidate also completes Scriptorium's unchanged full functional +suite at clean consumer commit `df8039cc252e2e48772ed88b9d273b98e30a67c5`: +42 suites, 41 functional passes and the sole deliberate failure-propagation +fixture produces exit 1, using `--execution-timeout 1200`. Its local consumer +log `target/full-cli-budget-reviewed-red.log` spans about 202 seconds. This +particular rerun does not exceed 300 seconds. The earlier initial-candidate +consumer run at `2d1d6e2` plus command/docs changes spans 509 seconds with the +same 41-pass/1-deliberate-failure outcome, and the final runtime's dedicated +305-second WFL test separately verifies the longer-boundary behavior. These +host-dependent run durations are evidence, not a performance benchmark. + +The proxy fixture's port race is remedied without retries or weakened tests: +its WFL server binds port zero and reports the actual bound address, and the +existing Rust harness reads a complete readiness record and validates loopback +address/nonzero port before connecting. All seven existing test bodies, +assertions, security settings and cleanup remain unchanged. The updated fixture +passes 7/7 locally at the final product source in the normal debug harness; +the release candidate is unchanged. This edits an existing fixture only and +adds no Rust test scenarios or test drivers. diff --git a/tests/trusted_proxy_test.rs b/tests/trusted_proxy_test.rs index 269354b3..d4082f93 100644 --- a/tests/trusted_proxy_test.rs +++ b/tests/trusted_proxy_test.rs @@ -29,7 +29,6 @@ impl Server { async fn start_with_handler(proxies: &str, tls: bool, handler: &str) -> Self { let directory = tempfile::tempdir().unwrap(); - let port = common::free_tcp_port(); std::fs::write( directory.path().join(".wflcfg"), format!( @@ -52,7 +51,8 @@ impl Server { std::fs::write( directory.path().join("server.wfl"), format!( - r#"listen on port {port}{secured} as server_handle + r#"listen on port 0{secured} as server_handle +display "TRUSTED_PROXY_READY " with server_handle main loop: wait for request comes in on server_handle as req with timeout 10000 check if path is equal to "/shutdown": @@ -87,7 +87,7 @@ end loop let mut server = Self { child, directory, - url: format!("{}://127.0.0.1:{port}", if tls { "https" } else { "http" }), + url: String::new(), client: reqwest::Client::builder() .no_proxy() .danger_accept_invalid_certs(tls) @@ -102,11 +102,24 @@ end loop "server exited before readiness: {}", server.log() ); - if tokio::net::TcpStream::connect(("127.0.0.1", port)) - .await - .is_ok() + // The child owns its OS-assigned port before publishing it. Reading + // only complete log lines avoids accepting a partial readiness write. + let log = server.log(); + if let Some(address) = log + .split_inclusive('\n') + .filter(|line| line.ends_with('\n')) + .find_map(|line| { + line.trim_end() + .strip_prefix("TRUSTED_PROXY_READY WebServer::") + }) { - break; + let address: std::net::SocketAddr = address.parse().expect("bound server address"); + assert_eq!(address.ip(), std::net::Ipv4Addr::LOCALHOST); + assert_ne!(address.port(), 0, "server must report its assigned port"); + server.url = format!("{}://{address}", if tls { "https" } else { "http" }); + if tokio::net::TcpStream::connect(address).await.is_ok() { + break; + } } assert!( Instant::now() < deadline,