diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1997465..79343ef7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -313,6 +313,13 @@ jobs: RUST_MIN_STACK: '8388608' run: ./scripts/run_integration_tests.ps1 + # Real ordinary execution beyond the historical 300-second CLI ceiling. + # The complete scenario/assertions are WFL; this step only invokes it. + # It cannot use the recursive program sweep's 30-second per-file bound. + - name: Verify explicit execution budget beyond five minutes + timeout-minutes: 6 + run: ./target/release/wfl --execution-timeout 330 --test tests/fixtures/cli_budget/long-run.test.wfl + # Validate that documentation examples still parse/analyze/lint against the # current release binary (testing.md requires docs validation in CI). # `--force` ignores the committed cache so CI always re-validates rather diff --git a/CLAUDE.md b/CLAUDE.md index 25355713..869ee11b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,6 +146,7 @@ Source Code → Lexer → Parser → Analyzer → Type Checker → Interpreter - `wfl --dump-env`: Dump environment for troubleshooting. - `wfl --analyze `: Run static analysis. - `wfl --test `: Run file in test mode (executes describe/test blocks). +- `wfl --execution-timeout `: Override only this invocation's shared deadline; whole seconds from 1 through 31536000, before the filename. Defaults, config caps and other limits remain unchanged. ## Key Language Features - **Natural Language Syntax**: `store name as "value"`, `check if x is greater than 5`. diff --git a/Docs/reference/configuration-reference.md b/Docs/reference/configuration-reference.md index 338f215e..96d5075e 100644 --- a/Docs/reference/configuration-reference.md +++ b/Docs/reference/configuration-reference.md @@ -290,6 +290,40 @@ so a stalled remote peer cannot wedge the server indefinitely. - **Default:** `60` - **Example:** `timeout_seconds = 300` +The file CLI preserves a maximum of 300 seconds for this configuration setting. +For a trusted batch job that needs a longer finite invocation, place +`--execution-timeout SECONDS` before its filename: + +```bash +wfl --execution-timeout 1200 batch.wfl +wfl --execution-timeout 1200 --test batch.test.wfl +``` + +The option accepts whole seconds from 1 through 31,536,000 (one year). Zero, +negative or fractional values, nonnumeric values and duplicate options are +errors; there is no unlimited value. Without the option, the default remains +60 seconds and configured values retain the 300-second cap. The option may also +bound source analysis, lexing and parsing. It cannot accompany configuration +maintenance, environment dumps or editor launch. After an executable source +filename, arguments are passed literally to that program instead. + +This is an invocation-only override of the shared execution deadline, starting +before source loading and covering the front end, interpreter, includes and +executed files. It does not reset between tests or nested files, and it does not +implicitly change the configuration of separately launched WFL children. +Ordinary foreground subprocess operations continue sharing the invocation +deadline, while explicit process-wait timeouts remain independently enforced. +Duration waits observe cancellation and the invocation deadline while waiting, +including a final wait with no following statement. Waits within an active +`main loop` retain the server lifetime exemption. + +The override does not change other resource limits, subprocess permissions, +request/stream limits, or configured per-operation timeouts. A server's `main +loop` retains its existing lifetime exemption; its outbound requests and +implicit subprocess waits keep their finite configured timeouts. Use the option +only when the caller deliberately grants the job more execution time; it does +not establish a sandbox for untrusted programs. + #### `logging_enabled` Enables logging output to `wfl.log` in the script’s directory. diff --git a/Engineering/evidence/2026-09-20-cli-execution-budget.md b/Engineering/evidence/2026-09-20-cli-execution-budget.md new file mode 100644 index 00000000..ccaa2868 --- /dev/null +++ b/Engineering/evidence/2026-09-20-cli-execution-budget.md @@ -0,0 +1,207 @@ +# Finite invocation execution budgets + +Risk class: **R3** (resource limits, CLI compatibility and subprocess lifecycle). + +The Scriptorium full Linux suite reached the existing CLI's 300-second execution +deadline on official WFL 26.9.14 after about 34 suites, then subsequent child +launches failed against the same expired budget. The configured runner timeout +could not extend this limit. The same published runtime completed the Windows +consumer suite at `07e8adcb` in about 160 seconds: 41 suites passed and its one +deliberately failing gate test correctly made the invocation exit 1. That local +consumer log is `target/full-official-windows-red.log` in Scriptorium. This is a +real capacity failure, not a +reason to hide suite failures or exempt a batch runner with `main loop`. +The consumer Red is [Scriptorium CI35507634634](https://github.com/WebFirstLanguage/Scriptorium/actions/runs/35507634634) +at `07e8adcb`, using the published runtime rather than a development build. + +The published source `8d82d785ea59300834de1c48b04a7ba0e187a1cd` +unconditionally applies `timeout_seconds.min(300)` in `src/main.rs`. +Configuration accepts larger values, but there is no existing file-entrypoint +override. The July 10 bind-address diary records retaining the historical cap +for compatibility. The new option leaves that default/config behavior intact. + +## Red evidence + +New scenarios, fixtures and drivers are WFL. Before implementation, the three +`TestPrograms/cli_budget/*.test.wfl` programs were run with the official Windows +26.9.14 executable (SHA-256 +`da109d5926f6af4a2f24c45150140764c406c055aef2dd7e43e45b85278f1dfe`). +The argument suite failed 5/5 expectations, deadlines passed its unchanged +configuration baseline and failed the four override cases, and server-policy +failed its one override case. All programs parsed and ran as WFL tests. The new +flag is unavailable on that release; these are CLI capability failures, not +claims that the old runtime implemented the new flag incorrectly. The existing +consumer 300-second failure establishes the underlying semantic limitation. + +The initial draft had an invalid reserved variable name and expected the wrong +timeout wording. Those fixture mistakes were corrected before the recorded Red +run; they are not counted as product failures. Logs are retained only under +ignored `target/reports/cli-budget/red-*.log`. + +## Acceptance and design + +`--execution-timeout SECONDS`, before the source filename, selects one finite +invocation deadline from 1 through 31,536,000 whole seconds. It changes only +`BudgetLimits.max_duration` before the shared budget starts. It does not change +`WflConfig`, default/config caps, request/stream limits, explicit subprocess wait +timeouts, subprocess permissions, operation/depth/size ceilings, or separately +launched WFL child configuration. Ordinary foreground operations continue to +share the invocation deadline; their duration therefore grows with an explicit +larger invocation budget. Included and executed files share that same deadline. + +Fast WFL suites cover operands, routing, test mode, cumulative deadlines, child +configuration, parent-expiration cleanup, and unchanged main-loop HTTP timeout, +operation limits and subprocess permissions. +The cleanup test first proves the child can write under its own longer budget. +`tests/fixtures/cli_budget/long-run.test.wfl` is a real 305-second ordinary run, +invoked explicitly by both integration jobs with a 330-second budget. +It belongs outside the recursive 30-second program sweep and must not be skipped. + +## Green verification and review + +Red commit `84cb272c` preserves the original executable capability tests before +the product implementation. Green adds the CLI parsing and budget selection in +`src/main.rs`; no interpreter or configuration policy implementation changes. +Further cases cover disallowed CLI modes and unchanged operation/permission +limits. While bringing the fixtures to Green, the shared-file fixture gained +the required `execute file at` syntax and the child assertion fixture gained +`--test`. These fixture corrections are not product defects or semantic Red +evidence. The later resource-policy cases also fail against the official runtime +because the flag is absent, rather than because the old policies are incorrect. + +The release candidate reports version 26.9.15 and has SHA-256 +`1185f150c8214d982a27431d4b88b94f7f20d6ce6c718161c35120deb817650f`. +Local Windows verification at the reviewed source: + +- `cargo build --release --locked`: passed. +- Four fast WFL suites: arguments 6/6, deadlines 5/5, server policy 1/1, + resource policy 3/3; all 15 passed. +- `wfl --execution-timeout 330 --test tests/fixtures/cli_budget/long-run.test.wfl`: + passed 1/1 after the actual 305-second wait and checkpoint. This run was not + shortened, skipped or made lifetime-exempt. +- Existing Rust compatibility suites: 139/139 passed across execution budget, + CLI help/version, lint CLI, bind-address CLI, outbound HTTP budget, subprocess, + subprocess cleanup and subprocess security. +- `cargo fmt --all -- --check` and + `cargo clippy --all-targets --all-features -- -D warnings`: passed. +- Existing documentation validation: 36/36 passed. +- Static repository hygiene after staging all new files, and `git diff --check`: + passed. +- `cargo check --locked --manifest-path fuzz/Cargo.toml -j 1 --verbose`: + passed, including `fuzz_frontend`, in 5m22s. This is compilation evidence, + not a sustained fuzz campaign. + +The first default-parallel fuzz compilation failed while compiling unchanged +`crypto-common 0.2.2`: rustc exited 1 without an explaining compiler diagnostic. +The successful single-worker diagnostic build establishes that the candidate +fuzz workspace compiles, but does not identify the initial failure's cause. +Both logs are retained and the unresolved observation is tracked in +[issue #740](https://github.com/WebFirstLanguage/wfl/issues/740). No dependency, +lockfile or source change was made between those builds; the issue does not +waive any check. The host used stable x86_64-pc-windows-msvc Rust 1.98.1. + +Logs remain under ignored `target/reports/cli-budget/`. An independent reviewer +checked flag operands and placement, CLI modes and child argv, shared deadlines, +configured HTTP/operation/permission policies and cancellation cleanup. The +review's two fixture findings were fixed: the owned child now has its own longer +budget plus a positive control, and the child assertion program is launched in +test mode. Final source and fixture review identified no blocking finding. + +Exact-head CI must still complete, including both real five-minute boundary +steps and the ordinary Linux/Windows program sweeps. No merge or release is +authorized by this evidence record itself. + +## Review follow-up Red + +Automated review of `68c46650` exposed three additional semantic cases, verified +locally before their fixes in `TestPrograms/cli_budget/review-boundaries.test.wfl`. +The WFL suite passed its existing main-loop wait exemption baseline and failed +three real expectations: an ordinary five-second wait outlived a one-second +deadline, a one-second invocation override shortened a ten-second server HTTP +policy, and dump modes accepted a misplaced timeout after the source filename. +The log is `target/reports/cli-budget/red-review-boundaries.log` (1/4 passed). +The wait gap predates the CLI option; it matters to an explicit finite deadline +and cannot be hidden by adding a later operation checkpoint to the assertion. +These findings supersede the earlier source-review verdict until corrected. +An additional last-statement wait case then confirmed successful exit after an +expired deadline without any following checkpoint; the expanded Red was 1/5 +(`red-review-boundaries-final-wait.log`). Existing embedded-runtime tests also +establish that a manually supplied 250ms budget bounds main-loop HTTP requests, +so the remedy must distinguish a CLI invocation override without changing that +existing API contract. + +Additional WFL streamed-response tests failed 0/2 on the frozen `68c46650` +binary, covering delayed headers and delayed body reads under config 10s / CLI +1s. Their first draft used a reserved parameter name and an ungrouped list +expression; those fixture parse errors were corrected before the recorded +semantic Red. An independently authored resource suite also failed its ordinary +wait and active WebSocket wait cases while passing the main-loop exemption +baseline (1/3). Red commits are `01416f4d`, `370ec22a`, `a5988f13`, and `028ef59e`. + +The final remedy retains the original main-loop operation duration privately in +`ExecutionBudget`. Only `with_invocation_timeout` replaces run lifetime; existing +explicit-budget constructors retain their behavior. HTTP continues choosing the +minimum of that operation duration and the configured/remaining stream limit. +Duration waits now check eagerly before/after the wait and each WebSocket pump +iteration. Passive sleeping and receiving use at most 10ms intervals; handler +dispatch is awaited normally so WFL finally blocks and interpreter state unwind. +A reviewed intermediate outer-select design was rejected because dropping an +arbitrary running handler could skip that cleanup. Its successful checks are +not final-source acceptance evidence. Dump modes scan only for a misplaced new +option, preserving the handling of unrelated trailing arguments. + +The resource regression was strengthened in `5d781336` after review found that +driver-side process reaping/closing could mask a surviving descendant. The +fixture now establishes writer readiness, releases it after the owner's +deadline, and observes the marker before any parent poll/reap. Exact-port +WebSocket rebind also happens before the driver's cleanup. The final frozen +old-binary result is 1/3: the late-write assertion fails with an actual pre-reap +write, the ordinary WebSocket wait outlives the deadline, and the exempt server +case passes. The unchanged final-source suite passes 3/3 in about eight seconds. +This covers an idle active WebSocket receiver; queued handler traffic remains +covered by the existing Rust WebSocket suite, not by this new WFL fixture. + +The final reviewed no-drop candidate SHA-256 is +`c0619c544b09551ce7988f58a564f50ff04e48a5e94a6f903c08a141b911c516`. +At this source, all seven fast WFL suites pass 25/25. `cargo test --all --locked` +passes 2,414 tests, with 27 existing ignored tests across 175 result records; +this includes the unchanged custom-250ms main-loop HTTP contract, stream tests, +WebSocket tests and CLI compatibility. Release build, fmt, strict Clippy, +fuzz-target compilation, and documentation validation (36/36) pass. Local logs +are the `*-final*` and `wait-resources-green.log` files under the same ignored +report directory. Final source review found no remaining blocking issue. + +The first remote run for `68c46650`, +[CI 35509162373](https://github.com/WebFirstLanguage/wfl/actions/runs/35509162373), +is not acceptance evidence: Windows integration failed in unchanged +`trusted_proxy_test::default_and_untrusted_peers_ignore_forged_forwarding` when +binding port 53684 reported Windows address-in-use error 10048. Its Linux +integration sibling was then cancelled, so neither long-boundary step ran. +Both ordinary program sweeps and the other completed jobs passed. The fixture +obtains a free port before spawning the server, which leaves a reuse window; +the observed log does not establish who occupied it. The final local full Rust +run passes that test unchanged. A new exact-head CI run must pass all gates; +the earlier failure is retained rather than treated as a waiver. + +The final no-drop candidate also passes the actual 305-second WFL boundary +1/1 with `--execution-timeout 330`; `green-final-long.log` records the result. + +The final candidate also completes Scriptorium's unchanged full functional +suite at clean consumer commit `df8039cc252e2e48772ed88b9d273b98e30a67c5`: +42 suites, 41 functional passes and the sole deliberate failure-propagation +fixture produces exit 1, using `--execution-timeout 1200`. Its local consumer +log `target/full-cli-budget-reviewed-red.log` spans about 202 seconds. This +particular rerun does not exceed 300 seconds. The earlier initial-candidate +consumer run at `2d1d6e2` plus command/docs changes spans 509 seconds with the +same 41-pass/1-deliberate-failure outcome, and the final runtime's dedicated +305-second WFL test separately verifies the longer-boundary behavior. These +host-dependent run durations are evidence, not a performance benchmark. + +The proxy fixture's port race is remedied without retries or weakened tests: +its WFL server binds port zero and reports the actual bound address, and the +existing Rust harness reads a complete readiness record and validates loopback +address/nonzero port before connecting. All seven existing test bodies, +assertions, security settings and cleanup remain unchanged. The updated fixture +passes 7/7 locally at the final product source in the normal debug harness; +the release candidate is unchanged. This edits an existing fixture only and +adds no Rust test scenarios or test drivers. diff --git a/History/dev-diary/2026/2026-09-20-cli-execution-budget.md b/History/dev-diary/2026/2026-09-20-cli-execution-budget.md new file mode 100644 index 00000000..c43f9f79 --- /dev/null +++ b/History/dev-diary/2026/2026-09-20-cli-execution-budget.md @@ -0,0 +1,38 @@ +# A finite execution budget for long batch invocations + +The complete Scriptorium WFL suite exceeded the CLI's historical 300-second +cap on Linux. Increasing `.wflcfg` could not help because the cap was applied +after loading configuration. Making the test runner a lifetime-exempt server +loop would hide the boundary instead of supporting legitimate batch work. + +`wfl --execution-timeout 1200 batch.wfl` now grants one explicit finite deadline +to that invocation. Only `BudgetLimits.max_duration` changes; the interpreter +still receives its original capped configuration. This separation preserves +HTTP and streaming timeouts, main-loop process waits, explicit process-wait +deadlines, permissions and memory/operation/depth limits. Ordinary foreground +operations remain part of the same invocation budget, and executed files share +the same budget rather than resetting it. Child WFL processes retain their own +configuration unless the caller explicitly passes an override to them. + +The option accepts whole seconds from one through one year, rejects duplicate +or malformed operands, and must precede the source filename. Normal scripts and +test-mode scripts use the same option. Configuration maintenance, editor launch +and environment dumps refuse an irrelevant override instead of ignoring it. + +All new scenarios and drivers are WFL. Fast tests cover CLI routing, the existing +configuration baseline, longer/shorter deadlines, child cleanup and isolation, +and unchanged HTTP limits. A separate 305-second WFL case in both integration +jobs proves execution past the old ceiling with an explicit 330-second bound. +The existing outer CI bound remains finite. See the matching engineering +evidence record for Red/Green results, review and exact-head CI. + +Review exposed two policy boundaries that needed stronger tests: a shorter +invocation override must not shorten main-loop HTTP or streamed responses, and +a duration wait must observe the deadline even when it is the final statement. +The budget now retains its original operation duration separately when the CLI +overrides invocation lifetime, preserving existing embedded custom budgets. +Duration waits check eagerly and poll passive sleeping/receiving in bounded +intervals. WebSocket handlers are awaited normally so errors run their cleanup +and unwind interpreter state; the runtime does not cancel a handler future to +interrupt a duration wait. Dump modes reject the misplaced new option while +preserving their handling of unrelated trailing arguments. diff --git a/README.md b/README.md index 6dd45f32..5845818e 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Other projects can run their WFL test scripts with the | `wfl --lint ` | Lint; add `--fix --in-place` to auto-fix | | `wfl --analyze ` | Static analysis | | `wfl --test ` | Run `describe`/`test` blocks | +| `wfl --execution-timeout 1200 ` | Give this invocation a finite 20-minute execution budget; preserve other limits | | `wfl --parse ` / `wfl --lex ` | Dump the AST / tokens | ## Documentation diff --git a/TestPrograms/cli_budget/.wflcfg b/TestPrograms/cli_budget/.wflcfg new file mode 100644 index 00000000..39c47274 --- /dev/null +++ b/TestPrograms/cli_budget/.wflcfg @@ -0,0 +1,7 @@ +# Trusted CLI conformance drivers own only synthetic child programs. +timeout_seconds = 60 +execution_logging = false +debug_report_enabled = false +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true diff --git a/TestPrograms/cli_budget/arguments.test.wfl b/TestPrograms/cli_budget/arguments.test.wfl new file mode 100644 index 00000000..9c7d99d2 --- /dev/null +++ b/TestPrograms/cli_budget/arguments.test.wfl @@ -0,0 +1,72 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" +store case_root as path_join of repo_root and "target/test-artifacts/cli-budget-arguments" and (generate_uuid) +call makedirs with case_root + +describe "Explicit execution budget CLI arguments": + teardown: + call remove_dir with case_root and yes + end teardown + + test "the finite maximum is accepted without changing script arguments": + store source_path as path_join of fixture_root and "arguments.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "31536000", source_path, "literal spaces", "--execution-timeout", "0"] as outcome + expect outcome["exit_code"] to equal 0 + expect outcome["output"] to contain "ARG:literal spaces" + expect outcome["output"] to contain "ARG:--execution-timeout" + expect outcome["output"] to contain "ARG:0" + end test + + test "invalid operands fail before executing the source": + store source_path as path_join of fixture_root and "marker.wfl" + store marker_path as path_join of case_root and "must-not-exist.txt" + for each invalid_value in ["0", "-1", "1.5", "nan", "inf", "31536001", "18446744073709551616", "", "+1"]: + wait for execute command runtime_path with arguments ["--execution-timeout", invalid_value, source_path, marker_path] as outcome + expect outcome["exit_code"] to equal 2 + expect outcome["error"] to contain "--execution-timeout requires a whole number of seconds from 1 to 31536000" + expect (is_file of marker_path) to equal no + end for + end test + + test "missing operands and duplicate overrides are actionable errors": + wait for execute command runtime_path with arguments ["--execution-timeout"] as missing_outcome + expect missing_outcome["exit_code"] to equal 2 + expect missing_outcome["error"] to contain "--execution-timeout requires a whole number" + store source_path as path_join of fixture_root and "arguments.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "2", "--execution-timeout", "3", source_path] as duplicate + expect duplicate["exit_code"] to equal 2 + expect duplicate["error"] to contain "--execution-timeout may be specified only once" + end test + + test "test mode works in either prefix option order": + store source_path as path_join of fixture_root and "test-mode.test.wfl" + for each option_list in [["--test", "--execution-timeout", "3", source_path], ["--execution-timeout", "3", "--test", source_path]]: + wait for execute command runtime_path with arguments option_list as outcome + expect outcome["exit_code"] to equal 0 + expect outcome["output"] to contain "TEST MODE EXECUTED" + expect outcome["output"] to contain "Failed: 0" + end for + end test + + test "source-consuming modes reject an override after the filename": + store source_path as path_join of fixture_root and "arguments.wfl" + for each mode_option in ["--analyze", "--lint"]: + wait for execute command runtime_path with arguments [mode_option, source_path, "--execution-timeout", "3"] as outcome + expect outcome["exit_code"] to equal 2 + expect outcome["error"] to contain "--execution-timeout must appear before the source filename" + end for + end test + + test "non-execution operations cannot silently ignore the budget": + store source_path as path_join of fixture_root and "arguments.wfl" + for each mode_option in ["--configCheck", "--configFix", "--edit", "--dump-env"]: + wait for execute command runtime_path with arguments ["--execution-timeout", "3", mode_option, source_path] as outcome + expect outcome["exit_code"] to equal 2 + expect outcome["error"] to contain "--execution-timeout requires source execution or analysis" + end for + end test +end describe diff --git a/TestPrograms/cli_budget/deadlines.test.wfl b/TestPrograms/cli_budget/deadlines.test.wfl new file mode 100644 index 00000000..4296fc44 --- /dev/null +++ b/TestPrograms/cli_budget/deadlines.test.wfl @@ -0,0 +1,61 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" +store case_root as path_join of repo_root and "target/test-artifacts/cli-budget-deadlines" and (generate_uuid) +call makedirs with case_root + +describe "Invocation-only execution deadlines": + teardown: + call remove_dir with case_root and yes + end teardown + + test "configuration still governs ordinary invocations": + store source_path as path_join of fixture_root and "wait.wfl" + wait for execute command runtime_path with arguments [source_path, "1500"] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "COMPLETED") to equal no + end test + + test "an explicit budget extends this run beyond its configuration": + store source_path as path_join of fixture_root and "wait.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "4", source_path, "1500"] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "COMPLETED" + end test + + test "executed files share the cumulative invocation deadline": + store source_path as path_join of fixture_root and "shared-parent.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT COMPLETE") to equal no + end test + + test "expiration closes an owned child before it can write later": + store late_path as path_join of fixture_root and "late-marker.wfl" + store control_path as path_join of case_root and "control.txt" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", late_path, control_path] as control_outcome + expect control_outcome["success"] to equal yes + expect (is_file of control_path) to equal yes + store source_path as path_join of fixture_root and "owned-child.wfl" + store marker_path as path_join of case_root and "late.txt" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path, marker_path] as outcome + expect outcome["success"] to equal no + expect outcome["output"] to contain "OWNED CHILD STARTED" + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT COMPLETE") to equal no + wait for 3000 milliseconds + expect (is_file of marker_path) to equal no + end test + + test "separate WFL children retain their own configuration deadline": + store source_path as path_join of fixture_root and "child-config.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", "--test", source_path] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "CHILD CONFIGURATION RETAINED" + end test +end describe diff --git a/TestPrograms/cli_budget/resource-policy.test.wfl b/TestPrograms/cli_budget/resource-policy.test.wfl new file mode 100644 index 00000000..d5f1fb73 --- /dev/null +++ b/TestPrograms/cli_budget/resource-policy.test.wfl @@ -0,0 +1,32 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" + +describe "An execution duration override retains other ceilings": + test "a shorter explicit deadline overrides a longer configuration": + store source_path as path_join of fixture_root and "longer-config/wait.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path, "1500"] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "COMPLETED") to equal no + end test + + test "operation ceilings remain enforced during the invocation": + store source_path as path_join of fixture_root and "operations/loop.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", source_path] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "operation budget (100 operations)" + expect (outcome["output"] contains "MUST NOT ESCAPE") to equal no + end test + + test "an execution override cannot enable denied subprocesses": + store source_path as path_join of fixture_root and "denied/process.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", source_path] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "Command blocked by security policy" + expect (outcome["output"] contains "MUST NOT ESCAPE") to equal no + end test +end describe diff --git a/TestPrograms/cli_budget/review-boundaries.test.wfl b/TestPrograms/cli_budget/review-boundaries.test.wfl new file mode 100644 index 00000000..a966d236 --- /dev/null +++ b/TestPrograms/cli_budget/review-boundaries.test.wfl @@ -0,0 +1,76 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" + +describe "Invocation budget review boundaries": + test "ordinary waits stop at the deadline without another checkpoint": + store source_path as path_join of fixture_root and "wait-without-checkpoint.wfl" + for each child_args in [[source_path], ["--execution-timeout", "1", source_path]]: + store started_at as current time in milliseconds + wait for execute command runtime_path with arguments child_args as outcome + store elapsed_ms as (current time in milliseconds) minus started_at + expect elapsed_ms is less than 3000 to equal yes + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT FINISH WAIT") to equal no + end for + end test + + test "a final duration wait still reports its expired deadline": + store source_path as path_join of fixture_root and "wait-last.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path] as outcome + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + end test + + test "server loop waits remain exempt from invocation lifetime": + store source_path as path_join of fixture_root and "main-loop-wait.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "SERVER WAIT RETAINED" + end test + + test "a shorter invocation budget does not shorten server HTTP policy": + store peer_path as path_join of fixture_root and "slow-peer.wfl" + wait for spawn command runtime_path with arguments [peer_path] as peer_process + try: + store startup_text as "" + store peer_url as "" + count from 1 to 200: + wait for read output from process peer_process as fresh_text + change startup_text to startup_text with fresh_text + for each output_line in (split startup_text by "\n"): + check if output_line starts with "READY ": + change peer_url to trim of (substring of output_line and 6 and ((length of output_line) minus 6)) + end check + end for + check if peer_url is not equal to "": + break + end check + wait for 10 milliseconds + end count + expect peer_url is not equal to "" to equal yes + store source_path as path_join of fixture_root and "longer-config/server-client.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path, peer_url] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "RECEIVED late response" + finally: + close process peer_process + end try + expect (process peer_process is running) to equal no + end test + + test "dump modes require the execution override before the filename": + store source_path as path_join of fixture_root and "arguments.wfl" + for each mode_option in ["--lex", "--ast", "--parse"]: + wait for execute command runtime_path with arguments [mode_option, source_path, "--execution-timeout", "3"] as misplaced + expect misplaced["exit_code"] to equal 2 + expect misplaced["error"] to contain "--execution-timeout must appear before the source filename" + wait for execute command runtime_path with arguments ["--execution-timeout", "3", mode_option, source_path] as correct + expect correct["success"] to equal yes + end for + end test +end describe diff --git a/TestPrograms/cli_budget/server-policy.test.wfl b/TestPrograms/cli_budget/server-policy.test.wfl new file mode 100644 index 00000000..eba256ff --- /dev/null +++ b/TestPrograms/cli_budget/server-policy.test.wfl @@ -0,0 +1,39 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" + +describe "Execution override preserves server operation limits": + test "a main-loop request retains the configured one-second timeout": + store peer_path as path_join of fixture_root and "slow-peer.wfl" + wait for spawn command runtime_path with arguments [peer_path] as peer_process + try: + store startup_text as "" + store peer_url as "" + count from 1 to 200: + wait for read output from process peer_process as fresh_text + change startup_text to startup_text with fresh_text + for each output_line in (split startup_text by "\n"): + check if output_line starts with "READY ": + change peer_url to trim of (substring of output_line and 6 and ((length of output_line) minus 6)) + end check + end for + check if peer_url is not equal to "": + break + end check + wait for 10 milliseconds + end count + expect peer_url is not equal to "" to equal yes + store client_path as path_join of fixture_root and "server-client.wfl" + wait for execute command runtime_path with arguments ["--execution-timeout", "10", client_path, peer_url] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT RECEIVE") to equal no + finally: + close process peer_process + end try + expect (process peer_process is running) to equal no + end test +end describe diff --git a/TestPrograms/cli_budget/stream-policy.test.wfl b/TestPrograms/cli_budget/stream-policy.test.wfl new file mode 100644 index 00000000..11b42184 --- /dev/null +++ b/TestPrograms/cli_budget/stream-policy.test.wfl @@ -0,0 +1,46 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget" + +define action called assert_stream_policy with parameters request_path: + store peer_path as path_join of fixture_root and "stream-peer.wfl" + wait for spawn command runtime_path with arguments [peer_path] as peer_process + try: + store startup_text as "" + store peer_url as "" + count from 1 to 200: + wait for read output from process peer_process as fresh_text + change startup_text to startup_text with fresh_text + for each output_line in (split startup_text by "\n"): + check if output_line starts with "READY ": + change peer_url to trim of (substring of output_line and 6 and ((length of output_line) minus 6)) + end check + end for + check if peer_url is not equal to "": + break + end check + wait for 10 milliseconds + end count + expect peer_url is not equal to "" to equal yes + store source_path as path_join of fixture_root and "longer-config/stream-client.wfl" + store request_url as peer_url with request_path + wait for execute command runtime_path with arguments ["--execution-timeout", "1", source_path, request_url] as outcome + expect outcome["success"] to equal yes + expect outcome["output"] to contain "RECEIVED late stream response" + finally: + close process peer_process + end try + expect (process peer_process is running) to equal no +end action + +describe "Short invocation overrides retain server streaming policy": + test "delayed response headers use the configured server timeout": + call assert_stream_policy with "/head" + end test + test "delayed response body uses the configured stream timeout": + call assert_stream_policy with "/body" + end test +end describe diff --git a/TestPrograms/cli_budget/wait-resources.test.wfl b/TestPrograms/cli_budget/wait-resources.test.wfl new file mode 100644 index 00000000..80728f5c --- /dev/null +++ b/TestPrograms/cli_budget/wait-resources.test.wfl @@ -0,0 +1,140 @@ +store runtime_path as call current_executable +check if (length of args) is greater than 0: + change runtime_path to args[0] +end check +store repo_root as path_dirname of (path_dirname of script_directory) +store fixture_root as path_join of repo_root and "tests/fixtures/cli_budget/wait-resources" +store case_root as path_join of repo_root and "target/test-artifacts/cli-budget-wait-resources" and (generate_uuid) +call makedirs with case_root + +define action called resource_read_text with parameters source_path: + open file at source_path for reading as input_file + try: + wait for store text_value as read content from input_file + return text_value + finally: + close file input_file + end try +end action + +define action called resource_wait_ready with parameters ready_path: + count from 1 to 200: + check if is_file of ready_path: + return yes + end check + wait for 10 milliseconds + end count + return no +end action + +define action called resource_rebind_port with parameters listener_port: + listen for websockets on port listener_port as rebound_server + close server rebound_server + return yes +end action + +define action called resource_release_writer with parameters release_path: + open file at release_path for writing as release_file + try: + wait for write content "release" into release_file + finally: + close file release_file + end try +end action + +describe "Duration waits release owned resources at their deadline": + teardown: + call remove_dir with case_root and yes + end teardown + + test "a duration wait timeout stops its ready child before a delayed write": + store writer_path as path_join of fixture_root and "delayed-writer.wfl" + store control_ready as path_join of case_root and "control.ready" + store control_marker as path_join of case_root and "control.late" + store control_release as path_join of case_root and "control.release" + // Positive control: the same writer creates its file after release. + call resource_release_writer with control_release + wait for execute command runtime_path with arguments ["--execution-timeout", "10", writer_path, control_ready, control_marker, control_release] as control_outcome + expect control_outcome["success"] to equal yes + expect is_file of control_ready to equal yes + expect resource_read_text of control_marker to equal "delayed writer survived" + + store ready_path as path_join of case_root and "owned.ready" + store marker_path as path_join of case_root and "owned.late" + store release_path as path_join of case_root and "owned.release" + store parent_path as path_join of fixture_root and "owned-duration-wait.wfl" + store started_at as current time in milliseconds + wait for spawn command runtime_path with arguments ["--execution-timeout", "1", parent_path, ready_path, marker_path, release_path] as parent_process + try: + expect resource_wait_ready of ready_path to equal yes + // Neither poll nor reap the owner before this observation: native + // process completion also closes its tree and could hide a leak. + // Release the ready writer after its owner's deadline. The old + // unbounded duration wait is still active throughout this window. + wait for 1500 milliseconds + call resource_release_writer with release_path + store late_before_reap as resource_wait_ready of marker_path + wait for process parent_process to complete with timeout 7 and read result as outcome + store elapsed_ms as (current time in milliseconds) minus started_at + display "Duration wait owner: elapsed=" with elapsed_ms with ", success=" with outcome["success"] with ", late_write_before_reap=" with late_before_reap + expect outcome["output"] to contain "READY CHILD BEFORE DURATION WAIT" + expect late_before_reap to equal no + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT FINISH OWNED DURATION WAIT") to equal no + finally: + close process parent_process + end try + end test + + test "an active WebSocket receiver times out and releases its exact port": + store ready_path as path_join of case_root and "websocket-timeout.ready" + store source_path as path_join of fixture_root and "websocket-duration-wait.wfl" + store started_at as current time in milliseconds + wait for spawn command runtime_path with arguments ["--execution-timeout", "1", source_path, ready_path, "ordinary"] as server_process + try: + expect resource_wait_ready of ready_path to equal yes + store listener_port as parse_json of (resource_read_text of ready_path) + expect listener_port is greater than 0 to equal yes + // Prove this is the child's live port, not an unrelated free port. + store refused_while_live as no + try: + call resource_rebind_port with listener_port + when error: + change refused_while_live to yes + expect error_message to contain "Failed to start websocket server" + end try + expect refused_while_live to equal yes + wait for process server_process to complete with timeout 7 and read result as outcome + store elapsed_ms as (current time in milliseconds) minus started_at + store rebound as resource_rebind_port of listener_port + display "WebSocket duration wait: elapsed=" with elapsed_ms with ", success=" with outcome["success"] with ", rebound=" with rebound + expect elapsed_ms is less than 3000 to equal yes + expect outcome["success"] to equal no + expect outcome["error"] to contain "timeout (1s)" + expect (outcome["output"] contains "MUST NOT FINISH WEBSOCKET WAIT") to equal no + expect rebound to equal yes + finally: + close process server_process + end try + end test + + test "a WebSocket receiver in a main loop keeps its exemption and releases its port": + store ready_path as path_join of case_root and "websocket-loop.ready" + store source_path as path_join of fixture_root and "websocket-duration-wait.wfl" + store started_at as current time in milliseconds + wait for spawn command runtime_path with arguments ["--execution-timeout", "1", source_path, ready_path, "main-loop"] as server_process + try: + expect resource_wait_ready of ready_path to equal yes + store listener_port as parse_json of (resource_read_text of ready_path) + wait for process server_process to complete with timeout 5 and read result as outcome + store elapsed_ms as (current time in milliseconds) minus started_at + expect outcome["success"] to equal yes + expect outcome["output"] to contain "EXEMPT WEBSOCKET WAIT FINISHED" + expect elapsed_ms is greater than or equal to 1400 to equal yes + expect resource_rebind_port of listener_port to equal yes + finally: + close process server_process + end try + end test +end describe diff --git a/src/exec/budget.rs b/src/exec/budget.rs index d2d288b0..aa7d0322 100644 --- a/src/exec/budget.rs +++ b/src/exec/budget.rs @@ -340,6 +340,9 @@ impl std::error::Error for BudgetExceeded {} #[derive(Debug)] pub struct ExecutionBudget { limits: BudgetLimits, + /// Original finite operation ceiling for deadline-exempt main-loop HTTP. + /// An explicit CLI invocation override changes the run lifetime only. + main_loop_duration: Option, started: Instant, cancelled: AtomicBool, /// Total interpreter operations charged. Also drives the clock-sampling @@ -377,6 +380,7 @@ impl ExecutionBudget { /// Build a budget from explicit limits, starting the deadline clock now. pub fn new(limits: BudgetLimits) -> Self { Self { + main_loop_duration: limits.max_duration, limits, started: Instant::now(), cancelled: AtomicBool::new(false), @@ -388,6 +392,20 @@ impl ExecutionBudget { } } + /// Override only the invocation lifetime, preserving the original finite + /// main-loop operation ceiling. Existing explicit-budget constructors keep + /// their historical shared lifetime/operation-duration behavior. + pub fn with_invocation_timeout(limits: BudgetLimits, duration: Duration) -> Self { + let mut budget = Self::new(limits); + budget.limits.max_duration = Some(duration); + budget + } + + /// Original operation duration, unaffected by a CLI lifetime override. + pub fn main_loop_duration(&self) -> Option { + self.main_loop_duration + } + /// Build a budget from a loaded configuration. See /// [`BudgetLimits::from_config`]. pub fn from_config(config: &WflConfig) -> Self { diff --git a/src/interpreter/mod.rs b/src/interpreter/mod.rs index cb86f7c1..f60068cf 100644 --- a/src/interpreter/mod.rs +++ b/src/interpreter/mod.rs @@ -3899,7 +3899,7 @@ impl IoClient { if budget.is_deadline_exempt() { return Ok(OutboundHttpDeadline::MainLoop { - duration: budget.limits().max_duration.unwrap_or(configured_timeout), + duration: budget.main_loop_duration().unwrap_or(configured_timeout), }); } @@ -9605,10 +9605,14 @@ impl Interpreter { // While WebSocket servers are running, spend the wait window // dispatching their events to the registered handler blocks. - // With no WebSocket servers this is an ordinary sleep, so the - // statement's timing semantics are unchanged. - self.pump_websocket_events(std::time::Duration::from_millis(duration_ms)) + // The pump checks eagerly and bounds its sleeping/receiving + // intervals, while awaiting WFL handlers normally so their + // finally blocks and interpreter frames always unwind. + self.check_wait_budget(*line, *column)?; + self.pump_websocket_events(Duration::from_millis(duration_ms)) .await?; + // A final wait must fail even without a later sampled check. + self.check_wait_budget(*line, *column)?; Ok((Value::Null, ControlFlow::None)) } Statement::TryStatement { @@ -13921,17 +13925,35 @@ impl Interpreter { )) } + /// Eager deadline/cancellation check for asynchronous duration waits. + fn check_wait_budget(&self, line: usize, column: usize) -> Result<(), RuntimeError> { + self.budget + .check_cancelled() + .map_err(|e| self.budget_error(e, line, column))?; + if !self.budget.is_deadline_exempt() { + self.budget + .check_deadline() + .map_err(|e| self.budget_error(e, line, column))?; + } + Ok(()) + } + /// Drains and dispatches queued websocket events for up to `budget`, running /// the matching handler block for each. With no websocket servers active it /// is a plain sleep, preserving `wait for ` semantics. async fn pump_websocket_events(&self, budget: Duration) -> Result<(), RuntimeError> { let deadline = tokio::time::Instant::now() + budget; loop { + // Check even when queued events or closed channels remain ready. + self.check_wait_budget(0, 0)?; let now = tokio::time::Instant::now(); if now >= deadline { break; } let remaining = deadline - now; + // Poll only the passive wait, not an arbitrary WFL handler future: + // dropping a handler could skip finally and leave action state. + let wait_slice = remaining.min(Duration::from_millis(10)); // Snapshot the receivers with a short borrow; dispatch below must not // hold a borrow of web_socket_servers across handler execution. @@ -13943,8 +13965,8 @@ impl Interpreter { .collect(); if receivers.is_empty() { - tokio::time::sleep(remaining).await; - break; + tokio::time::sleep(wait_slice).await; + continue; } let mut recv_futs = Vec::with_capacity(receivers.len()); @@ -13956,11 +13978,11 @@ impl Interpreter { })); } - let sleep_fut = tokio::time::sleep(remaining); + let sleep_fut = tokio::time::sleep(wait_slice); tokio::pin!(sleep_fut); tokio::select! { - _ = &mut sleep_fut => break, + _ = &mut sleep_fut => {}, ((key, event), _idx, _rest) = futures_util::future::select_all(recv_futs) => { if let Some(event) = event { self.dispatch_ws_event(&key, event).await?; diff --git a/src/main.rs b/src/main.rs index 8219a251..3827be59 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,6 +45,9 @@ fn print_help() { println!(" --output Specify an output file for the environment dump"); println!(" --time Measure and display execution time"); println!(" --test Run file in test mode"); + println!(" --execution-timeout "); + println!(" Set this invocation's finite execution budget (1–31536000)"); + println!(" Place before the source filename; other limits are unchanged"); println!(); println!("Configuration Maintenance:"); println!(" --configCheck Check configuration files for issues"); @@ -276,6 +279,7 @@ async fn run() -> io::Result<()> { let mut output_path = None; let mut time_mode = false; let mut test_mode = false; + let mut execution_timeout = None; let mut file_path = String::new(); let mut i = 1; @@ -292,6 +296,29 @@ async fn run() -> io::Result<()> { "--lint" | "--fix" | "--diff" | "--in-place" )); match args[i].as_str() { + "--execution-timeout" => { + if !file_path.is_empty() { + eprintln!("Error: --execution-timeout must appear before the source filename"); + process::exit(2); + } + if execution_timeout.is_some() { + eprintln!("Error: --execution-timeout may be specified only once"); + process::exit(2); + } + let seconds = args + .get(i + 1) + .filter(|value| !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit())) + .and_then(|value| value.parse::().ok()) + .filter(|seconds| (1..=31_536_000).contains(seconds)); + let Some(seconds) = seconds else { + eprintln!( + "Error: --execution-timeout requires a whole number of seconds from 1 to 31536000" + ); + process::exit(2); + }; + execution_timeout = Some(std::time::Duration::from_secs(seconds)); + i += 2; + } "--init" => { eprintln!("Error: initialization is a command. Use: wfl init"); process::exit(2); @@ -484,6 +511,20 @@ async fn run() -> io::Result<()> { // Validate the completed option set before running any operation or writing // output. Checking here makes conflicts independent of argument order. + // Dump modes do not execute script arguments, but historically ignore them. + // Reject only this misplaced new option without reinterpreting legacy argv. + if (lex_dump || ast_dump) && args[i..].iter().any(|arg| arg == "--execution-timeout") { + eprintln!("Error: --execution-timeout must appear before the source filename"); + process::exit(2); + } + if execution_timeout.is_some() + && (config_check_mode || config_fix_mode || edit_mode || dump_env_mode) + { + eprintln!( + "Error: --execution-timeout requires source execution or analysis; it cannot be combined with --configCheck, --configFix, --edit, or --dump-env" + ); + process::exit(2); + } if fix_diff && fix_in_place { eprintln!("Error: --in-place and --diff flags are mutually exclusive"); process::exit(2); @@ -622,14 +663,21 @@ async fn run() -> io::Result<()> { let script_dir = Path::new(&file_path).parent().unwrap_or(Path::new(".")); let config = config::load_config(script_dir); - // Build the ONE execution budget for this run up front, from the same - // (timeout-capped) config the interpreter will use, so a single budget - // governs the pre-parse source check, lexing/parsing/analysis, and - // interpretation — its deadline clock starts here and covers the whole run. + // Preserve the historic config timeout cap and its per-operation uses. + // An explicit invocation override changes ONLY the shared budget duration, + // not request/stream timeouts, main-loop subprocess waits, or other limits. + // The one deadline starts before reading/lexing/parsing/analysis and is + // reused by interpretation and nested executed files. let mut run_config = config.clone(); run_config.timeout_seconds = run_config.timeout_seconds.min(300); let run_config = std::sync::Arc::new(run_config); - let budget = std::sync::Arc::new(wfl::exec::budget::ExecutionBudget::from_config(&run_config)); + let budget_limits = wfl::exec::budget::BudgetLimits::from_config(&run_config); + let budget = std::sync::Arc::new(match execution_timeout { + Some(duration) => { + wfl::exec::budget::ExecutionBudget::with_invocation_timeout(budget_limits, duration) + } + None => wfl::exec::budget::ExecutionBudget::new(budget_limits), + }); // Install the run budget as the current-thread budget for the ENTIRE run, so // every front-end phase — lexing, parsing, analysis, type checking — and the @@ -1037,10 +1085,9 @@ async fn run() -> io::Result<()> { // Reuse the single budget (and the timeout-capped `run_config`) // built at the top of the run, so the source check and the // interpreter share one deadline/operation/cancellation budget. - // `run_config` carries the full `.wflcfg` (e.g. - // `web_server_bind_address`) with the 300s timeout cap applied - // (see issue #466); the cap never affects web servers because - // `check_time` skips the deadline inside a main loop. + // `run_config` preserves config policy, while `budget` may carry + // an explicit CLI execution duration. Main loops keep their + // existing lifetime exemption and finite operation timeouts. let mut interpreter = Interpreter::with_config_and_budget( std::sync::Arc::clone(&run_config), std::sync::Arc::clone(&budget), diff --git a/testing.md b/testing.md index 36c45a89..ecf61fa7 100644 --- a/testing.md +++ b/testing.md @@ -41,6 +41,7 @@ runs on Tokio. | Extension dependency security | `cd vscode-extension && npm ci && npm run test:security` | | Extension lint + VS Code host | `cargo build --locked -p wfl-lsp`, then `cd vscode-extension && npm ci && npm test` (use `xvfb-run -a npm test` on headless Linux) | | WFL end-to-end programs | `cargo build --release` then `./scripts/run_integration_tests.sh` (`.ps1` on Windows) | +| Long invocation boundary | `target/release/wfl --execution-timeout 330 --test tests/fixtures/cli_budget/long-run.test.wfl` (`wfl.exe` on Windows; about 305 seconds) | | Web-server end-to-end | `./scripts/run_web_tests.sh` (`.ps1` on Windows) | | Docs examples validation | `python scripts/validate_docs_examples.py` | | Benchmarks (perf, non-gating) | `cargo bench` | @@ -55,6 +56,7 @@ cargo fmt --all -- --check \ && (cd vscode-extension && npm ci && xvfb-run -a npm test) \ && cargo build --release \ && ./scripts/run_integration_tests.sh \ + && target/release/wfl --execution-timeout 330 --test tests/fixtures/cli_budget/long-run.test.wfl \ && ./scripts/run_web_tests.sh \ && python scripts/validate_docs_examples.py ``` @@ -126,6 +128,13 @@ Programs**, web tests, database tests, and fuzz-target compilation. All are required checks; the default branch MUST stay green. The nightly build is a release artifact and is separately monitored. +Both integration jobs also execute the WFL long-invocation boundary fixture +with a 330-second execution budget and a six-minute outer job-step limit. The +fixture asserts real ordinary execution beyond 300 seconds. The fast +`TestPrograms/cli_budget/` suites remain in the usual recursive program gates; +the long fixture lives under `tests/fixtures/` and is invoked explicitly rather +than skipped by the program sweep's 30-second limit. + The existing **Build, Test, Clippy** job also installs locked extension dependencies, runs the bounded dependency security regression, and runs the complete `npm test` command in a real VS Code host after building the LSP. diff --git a/tests/fixtures/cli_budget/.wflcfg b/tests/fixtures/cli_budget/.wflcfg new file mode 100644 index 00000000..2a95c9c7 --- /dev/null +++ b/tests/fixtures/cli_budget/.wflcfg @@ -0,0 +1,6 @@ +timeout_seconds = 1 +execution_logging = false +debug_report_enabled = false +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true diff --git a/tests/fixtures/cli_budget/arguments.wfl b/tests/fixtures/cli_budget/arguments.wfl new file mode 100644 index 00000000..2a5c9a4e --- /dev/null +++ b/tests/fixtures/cli_budget/arguments.wfl @@ -0,0 +1,3 @@ +for each argument_value in args: + display "ARG:" with argument_value +end for diff --git a/tests/fixtures/cli_budget/child-config.wfl b/tests/fixtures/cli_budget/child-config.wfl new file mode 100644 index 00000000..758dafee --- /dev/null +++ b/tests/fixtures/cli_budget/child-config.wfl @@ -0,0 +1,6 @@ +store runtime_path as call current_executable +store child_path as path_join of script_directory and "wait.wfl" +wait for execute command runtime_path with arguments [child_path, "1500"] as outcome +expect outcome["success"] to equal no +expect outcome["error"] to contain "timeout (1s)" +display "CHILD CONFIGURATION RETAINED" diff --git a/tests/fixtures/cli_budget/denied/.wflcfg b/tests/fixtures/cli_budget/denied/.wflcfg new file mode 100644 index 00000000..914ead42 --- /dev/null +++ b/tests/fixtures/cli_budget/denied/.wflcfg @@ -0,0 +1,4 @@ +timeout_seconds = 1 +allow_shell_execution = false +execution_logging = false +debug_report_enabled = false diff --git a/tests/fixtures/cli_budget/denied/process.wfl b/tests/fixtures/cli_budget/denied/process.wfl new file mode 100644 index 00000000..9b8a5c8f --- /dev/null +++ b/tests/fixtures/cli_budget/denied/process.wfl @@ -0,0 +1,3 @@ +store runtime_path as call current_executable +wait for execute command runtime_path with arguments ["--version"] as outcome +display "MUST NOT ESCAPE PROCESS POLICY" diff --git a/tests/fixtures/cli_budget/late-marker.wfl b/tests/fixtures/cli_budget/late-marker.wfl new file mode 100644 index 00000000..e81bb8ed --- /dev/null +++ b/tests/fixtures/cli_budget/late-marker.wfl @@ -0,0 +1,4 @@ +wait for 2500 milliseconds +open file at args[0] for writing as marker_file +wait for write content "late child survived" into marker_file +close file marker_file diff --git a/tests/fixtures/cli_budget/long-run.test.wfl b/tests/fixtures/cli_budget/long-run.test.wfl new file mode 100644 index 00000000..76faa4e0 --- /dev/null +++ b/tests/fixtures/cli_budget/long-run.test.wfl @@ -0,0 +1,14 @@ +// Invoked explicitly by both integration jobs with --execution-timeout 330. +// This real boundary test must not enter a lifetime-exempt main loop. +describe "Explicit execution budget above five minutes": + test "ordinary WFL execution can complete beyond the legacy cap": + store started_at as current time in milliseconds + wait for 305 seconds + count from 1 to 5000: + store observed as count + end count + store elapsed_ms as (current time in milliseconds) minus started_at + expect elapsed_ms to be greater than 300000 + display "LONG EXECUTION BUDGET VERIFIED" + end test +end describe diff --git a/tests/fixtures/cli_budget/longer-config/.wflcfg b/tests/fixtures/cli_budget/longer-config/.wflcfg new file mode 100644 index 00000000..edeb859c --- /dev/null +++ b/tests/fixtures/cli_budget/longer-config/.wflcfg @@ -0,0 +1,3 @@ +timeout_seconds = 10 +execution_logging = false +debug_report_enabled = false diff --git a/tests/fixtures/cli_budget/longer-config/server-client.wfl b/tests/fixtures/cli_budget/longer-config/server-client.wfl new file mode 100644 index 00000000..1640bb6e --- /dev/null +++ b/tests/fixtures/cli_budget/longer-config/server-client.wfl @@ -0,0 +1,5 @@ +main loop: + open url at args[0] and read content as response_text + display "RECEIVED " with response_text + exit program +end loop diff --git a/tests/fixtures/cli_budget/longer-config/stream-client.wfl b/tests/fixtures/cli_budget/longer-config/stream-client.wfl new file mode 100644 index 00000000..23be2738 --- /dev/null +++ b/tests/fixtures/cli_budget/longer-config/stream-client.wfl @@ -0,0 +1,10 @@ +main loop: + open url at args[0] and stream response as upstream + try: + wait for next line from upstream as response_text + display "RECEIVED " with response_text + finally: + close upstream + end try + exit program +end loop diff --git a/tests/fixtures/cli_budget/longer-config/wait.wfl b/tests/fixtures/cli_budget/longer-config/wait.wfl new file mode 100644 index 00000000..09a4243e --- /dev/null +++ b/tests/fixtures/cli_budget/longer-config/wait.wfl @@ -0,0 +1 @@ +include from "../wait.wfl" diff --git a/tests/fixtures/cli_budget/main-loop-wait.wfl b/tests/fixtures/cli_budget/main-loop-wait.wfl new file mode 100644 index 00000000..4e61f90c --- /dev/null +++ b/tests/fixtures/cli_budget/main-loop-wait.wfl @@ -0,0 +1,5 @@ +main loop: + wait for 1500 milliseconds + display "SERVER WAIT RETAINED" + exit program +end loop diff --git a/tests/fixtures/cli_budget/marker.wfl b/tests/fixtures/cli_budget/marker.wfl new file mode 100644 index 00000000..d4117279 --- /dev/null +++ b/tests/fixtures/cli_budget/marker.wfl @@ -0,0 +1,4 @@ +open file at args[0] for writing as marker_file +wait for write content "executed" into marker_file +close file marker_file +display "MARKER WRITTEN" diff --git a/tests/fixtures/cli_budget/operations/.wflcfg b/tests/fixtures/cli_budget/operations/.wflcfg new file mode 100644 index 00000000..27f8a61a --- /dev/null +++ b/tests/fixtures/cli_budget/operations/.wflcfg @@ -0,0 +1,4 @@ +timeout_seconds = 1 +max_operations = 100 +execution_logging = false +debug_report_enabled = false diff --git a/tests/fixtures/cli_budget/operations/loop.wfl b/tests/fixtures/cli_budget/operations/loop.wfl new file mode 100644 index 00000000..750aa319 --- /dev/null +++ b/tests/fixtures/cli_budget/operations/loop.wfl @@ -0,0 +1,4 @@ +count from 1 to 5000: + store observed as count +end count +display "MUST NOT ESCAPE OPERATION LIMIT" diff --git a/tests/fixtures/cli_budget/owned-child.wfl b/tests/fixtures/cli_budget/owned-child.wfl new file mode 100644 index 00000000..787a393c --- /dev/null +++ b/tests/fixtures/cli_budget/owned-child.wfl @@ -0,0 +1,6 @@ +store runtime_path as call current_executable +store child_path as path_join of script_directory and "late-marker.wfl" +wait for spawn command runtime_path with arguments ["--execution-timeout", "10", child_path, args[0]] as owned_child +display "OWNED CHILD STARTED" +wait for process owned_child to complete with timeout 10 and read result as outcome +display "MUST NOT COMPLETE OWNED WAIT" diff --git a/tests/fixtures/cli_budget/server-client.wfl b/tests/fixtures/cli_budget/server-client.wfl new file mode 100644 index 00000000..9dc4c7ae --- /dev/null +++ b/tests/fixtures/cli_budget/server-client.wfl @@ -0,0 +1,9 @@ +main loop: + try: + open url at args[0] and read content as content_value + display "MUST NOT RECEIVE LATE BODY" + when error: + display error_message + end try + break +end loop diff --git a/tests/fixtures/cli_budget/shared-child.wfl b/tests/fixtures/cli_budget/shared-child.wfl new file mode 100644 index 00000000..f729500a --- /dev/null +++ b/tests/fixtures/cli_budget/shared-child.wfl @@ -0,0 +1,5 @@ +wait for 700 milliseconds +count from 1 to 5000: + store observed as count +end count +display "MUST NOT COMPLETE CHILD" diff --git a/tests/fixtures/cli_budget/shared-parent.wfl b/tests/fixtures/cli_budget/shared-parent.wfl new file mode 100644 index 00000000..5853b19a --- /dev/null +++ b/tests/fixtures/cli_budget/shared-parent.wfl @@ -0,0 +1,3 @@ +wait for 700 milliseconds +execute file at "shared-child.wfl" +display "MUST NOT COMPLETE SHARED RUN" diff --git a/tests/fixtures/cli_budget/slow-peer.wfl b/tests/fixtures/cli_budget/slow-peer.wfl new file mode 100644 index 00000000..3dcdde35 --- /dev/null +++ b/tests/fixtures/cli_budget/slow-peer.wfl @@ -0,0 +1,12 @@ +listen on port 0 as slow_peer +store listener_text as "" with slow_peer +display "READY http://" with (substring of listener_text and 11 and ((length of listener_text) minus 11)) +main loop: + wait for request comes in on slow_peer as incoming + wait for 2500 milliseconds + try: + respond to incoming with "late response" + when error: + display "TIMED CLIENT DISCONNECTED" + end try +end loop diff --git a/tests/fixtures/cli_budget/stream-peer.wfl b/tests/fixtures/cli_budget/stream-peer.wfl new file mode 100644 index 00000000..a3a28cf2 --- /dev/null +++ b/tests/fixtures/cli_budget/stream-peer.wfl @@ -0,0 +1,22 @@ +listen on port 0 as stream_peer +store listener_text as "" with stream_peer +display "READY http://" with (substring of listener_text and 11 and ((length of listener_text) minus 11)) +main loop: + wait for request comes in on stream_peer as incoming + check if (path of incoming) is equal to "/head": + wait for 2500 milliseconds + end check + try: + start streaming response to incoming with status 200 and content type "text/plain" as outgoing + try: + check if (path of incoming) is equal to "/body": + wait for 2500 milliseconds + end check + write line "late stream response" to outgoing + finally: + close outgoing + end try + when error: + display "TIMED CLIENT DISCONNECTED" + end try +end loop diff --git a/tests/fixtures/cli_budget/test-mode.test.wfl b/tests/fixtures/cli_budget/test-mode.test.wfl new file mode 100644 index 00000000..16ff54ef --- /dev/null +++ b/tests/fixtures/cli_budget/test-mode.test.wfl @@ -0,0 +1,6 @@ +describe "Execution budget preserves test mode": + test "assertions actually execute": + expect 7 to equal 7 + display "TEST MODE EXECUTED" + end test +end describe diff --git a/tests/fixtures/cli_budget/wait-last.wfl b/tests/fixtures/cli_budget/wait-last.wfl new file mode 100644 index 00000000..f747568d --- /dev/null +++ b/tests/fixtures/cli_budget/wait-last.wfl @@ -0,0 +1 @@ +wait for 1500 milliseconds diff --git a/tests/fixtures/cli_budget/wait-resources/.wflcfg b/tests/fixtures/cli_budget/wait-resources/.wflcfg new file mode 100644 index 00000000..8a7a852c --- /dev/null +++ b/tests/fixtures/cli_budget/wait-resources/.wflcfg @@ -0,0 +1,7 @@ +timeout_seconds = 10 +execution_logging = false +debug_report_enabled = false +allow_shell_execution = true +shell_execution_mode = sanitized +kill_on_shutdown = true +web_server_bind_address = 127.0.0.1 diff --git a/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl b/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl new file mode 100644 index 00000000..04fb5c98 --- /dev/null +++ b/tests/fixtures/cli_budget/wait-resources/delayed-writer.wfl @@ -0,0 +1,17 @@ +open file at args[0] for writing as ready_file +wait for write content "ready" into ready_file +close file ready_file +store release_seen as no +count from 1 to 160: + check if is_file of args[2]: + change release_seen to yes + break + end check + wait for 50 milliseconds +end count +check if release_seen is equal to no: + call raise_error with "Delayed writer was never released" +end check +open file at args[1] for writing as marker_file +wait for write content "delayed writer survived" into marker_file +close file marker_file diff --git a/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl b/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl new file mode 100644 index 00000000..b03a515a --- /dev/null +++ b/tests/fixtures/cli_budget/wait-resources/owned-duration-wait.wfl @@ -0,0 +1,21 @@ +store runtime_path as call current_executable +store writer_path as path_join of script_directory and "delayed-writer.wfl" +wait for spawn command runtime_path with arguments ["--execution-timeout", "10", writer_path, args[0], args[1], args[2]] as owned_child +try: + store ready_seen as no + count from 1 to 100: + check if is_file of args[0]: + change ready_seen to yes + break + end check + wait for 10 milliseconds + end count + check if ready_seen is equal to no: + call raise_error with "Delayed writer did not signal readiness" + end check + display "READY CHILD BEFORE DURATION WAIT" + wait for 6500 milliseconds + display "MUST NOT FINISH OWNED DURATION WAIT" +finally: + close process owned_child +end try diff --git a/tests/fixtures/cli_budget/wait-resources/websocket-duration-wait.wfl b/tests/fixtures/cli_budget/wait-resources/websocket-duration-wait.wfl new file mode 100644 index 00000000..469320b8 --- /dev/null +++ b/tests/fixtures/cli_budget/wait-resources/websocket-duration-wait.wfl @@ -0,0 +1,20 @@ +listen for websockets on port 0 as waiting_server +store address_parts as split waiting_server by ":" +store listener_port as address_parts[(length of address_parts) minus 1] +open file at args[0] for writing as ready_file +wait for write content listener_port into ready_file +close file ready_file +try: + check if args[1] is equal to "main-loop": + main loop: + wait for 1500 milliseconds + display "EXEMPT WEBSOCKET WAIT FINISHED" + exit program + end loop + otherwise: + wait for 4500 milliseconds + display "MUST NOT FINISH WEBSOCKET WAIT" + end check +finally: + close server waiting_server +end try diff --git a/tests/fixtures/cli_budget/wait-without-checkpoint.wfl b/tests/fixtures/cli_budget/wait-without-checkpoint.wfl new file mode 100644 index 00000000..9474572f --- /dev/null +++ b/tests/fixtures/cli_budget/wait-without-checkpoint.wfl @@ -0,0 +1,2 @@ +wait for 5000 milliseconds +display "MUST NOT FINISH WAIT" diff --git a/tests/fixtures/cli_budget/wait.wfl b/tests/fixtures/cli_budget/wait.wfl new file mode 100644 index 00000000..9a665662 --- /dev/null +++ b/tests/fixtures/cli_budget/wait.wfl @@ -0,0 +1,7 @@ +store delay_ms as parse_json of args[0] +wait for delay_ms milliseconds +// Exercise deadline checkpoints after sleeping, before claiming completion. +count from 1 to 5000: + store observed as count +end count +display "COMPLETED" diff --git a/tests/trusted_proxy_test.rs b/tests/trusted_proxy_test.rs index 269354b3..d4082f93 100644 --- a/tests/trusted_proxy_test.rs +++ b/tests/trusted_proxy_test.rs @@ -29,7 +29,6 @@ impl Server { async fn start_with_handler(proxies: &str, tls: bool, handler: &str) -> Self { let directory = tempfile::tempdir().unwrap(); - let port = common::free_tcp_port(); std::fs::write( directory.path().join(".wflcfg"), format!( @@ -52,7 +51,8 @@ impl Server { std::fs::write( directory.path().join("server.wfl"), format!( - r#"listen on port {port}{secured} as server_handle + r#"listen on port 0{secured} as server_handle +display "TRUSTED_PROXY_READY " with server_handle main loop: wait for request comes in on server_handle as req with timeout 10000 check if path is equal to "/shutdown": @@ -87,7 +87,7 @@ end loop let mut server = Self { child, directory, - url: format!("{}://127.0.0.1:{port}", if tls { "https" } else { "http" }), + url: String::new(), client: reqwest::Client::builder() .no_proxy() .danger_accept_invalid_certs(tls) @@ -102,11 +102,24 @@ end loop "server exited before readiness: {}", server.log() ); - if tokio::net::TcpStream::connect(("127.0.0.1", port)) - .await - .is_ok() + // The child owns its OS-assigned port before publishing it. Reading + // only complete log lines avoids accepting a partial readiness write. + let log = server.log(); + if let Some(address) = log + .split_inclusive('\n') + .filter(|line| line.ends_with('\n')) + .find_map(|line| { + line.trim_end() + .strip_prefix("TRUSTED_PROXY_READY WebServer::") + }) { - break; + let address: std::net::SocketAddr = address.parse().expect("bound server address"); + assert_eq!(address.ip(), std::net::Ipv4Addr::LOCALHOST); + assert_ne!(address.port(), 0, "server must report its assigned port"); + server.url = format!("{}://{address}", if tls { "https" } else { "http" }); + if tokio::net::TcpStream::connect(address).await.is_ok() { + break; + } } assert!( Instant::now() < deadline,