diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5757a758..c159b92e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,6 +10,8 @@ concurrency: cancel-in-progress: true jobs: build: + env: + POLYWORLD_DEPS: ${{ github.workspace }}/.. strategy: fail-fast: false matrix: @@ -29,10 +31,9 @@ jobs: run: | cd .. nimby create - nimby install "${{ github.event.repository.name }}/${{ github.event.repository.name }}.nimble" nimby sync "${{ github.event.repository.name }}/nimby.lock" - name: Run release tests and all-game determinism - run: nim r -d:release -d:bassyNativeStrict tests/all.nim + run: nim r -d:release -d:bassyNativeStrict -d:replayEvents tests/all.nim - name: Test lockstep training API run: nim r -d:release -d:headless -d:bassyNativeStrict tests/test_gota_lockstep.nim - name: Check neural converters diff --git a/coworld/dependencies.lock b/coworld/dependencies.lock index 808097e1..0ebdc672 100644 --- a/coworld/dependencies.lock +++ b/coworld/dependencies.lock @@ -6,7 +6,7 @@ silky 0.2.0 https://github.com/treeform/silky 3c5dd562924c044bfd4417331f82132fa0 pixie 6.1.0 https://github.com/treeform/pixie 30bb510877f02cbf9643e4f904e33d1c90f7910f vmath 3.0.0 https://github.com/treeform/vmath bc260143102153b76328783e1c76c96e52c52a4b chroma 1.0.0 https://github.com/treeform/chroma 08b0d65590221bb964544fd9badf29a146b9091a -zippy 0.10.20 https://github.com/guzba/zippy e41dfa36a19d739a2be7493211decadbe451031d +zippy 0.10.20 https://github.com/relh/zippy d605465c98b9666f9a5232cece5a0441c10d159e flatty 0.4.0 https://github.com/treeform/flatty 9c3f14cc287890b742196b4f0f74f0b9faab29cb nimsimd 1.3.2 https://github.com/guzba/nimsimd dd27017fbfd1c6c00d37776a8f5d322bf6c5a83b bumpy 1.1.3 https://github.com/treeform/bumpy 93583f6f8056a6259aaa96c89f4c143ba55bc6f0 diff --git a/coworld/integration.md b/coworld/integration.md index 71dc4f24..3588131b 100644 --- a/coworld/integration.md +++ b/coworld/integration.md @@ -131,6 +131,9 @@ It uses the binaries in `tmp/coworld` to check extensionless and empty sources, slot-specific print and annotation output, optional annotation destinations, compilation failure, disabled VMs, health/Ping/Pong, completion ordering and the 10 MiB log bound. +GotA also checks aggregate decoded-package limits and forged ZIP size metadata. +Set `COWORLD_TEST_ARTIFACTS` to an existing directory to retain each episode's +inputs, logs, status, results and replay for independent inspection. `nim r coworld/tools/test_tools.nim` checks concurrent build subprocesses, working-directory restoration, and failure logs. diff --git a/coworld/tools/test_runtime.nim b/coworld/tools/test_runtime.nim index 573f8a42..00ee5ac0 100644 --- a/coworld/tools/test_runtime.nim +++ b/coworld/tools/test_runtime.nim @@ -113,9 +113,15 @@ proc episode( ) = ## Runs one local roster and inspects outputs at the completion marker. doAssert scripts.len == count - let directory = createTempDir("coworld-" & game & "-", "") + let + evidenceRoot = getEnv("COWORLD_TEST_ARTIFACTS") + directory = createTempDir("coworld-" & game & "-", "", + if evidenceRoot.len > 0: evidenceRoot else: getTempDir()) defer: - removeDir(directory) + if evidenceRoot.len == 0: + removeDir(directory) + else: + echo "Runtime evidence: ", directory let listener = newSocket() listener.bindAddr(Port(0), "127.0.0.1") let port = listener.getLocalAddr()[1] @@ -388,6 +394,28 @@ print "PACKAGE-PASSED" episode("gota", 10, scripts, failure = true) echo "GOTA extensionless ZIP, mixed roster and package failures passed" + # A small upload must not bypass the aggregate decoded-package budget. + scripts[0] = zipFixture([("policy.bas", "end"), + ("first.bin", repeat('x', MaxUnpackedBytes div 2)), + ("second.bin", repeat('y', MaxUnpackedBytes div 2))], true) + episode("gota", 10, scripts, failure = true, + expectedRuntimeError = "Uncompressed data exceeds output limit") + + # Both directory and local sizes lie; enforcement must count actual output. + scripts[0] = zipFixture([("policy.bas", "end"), + ("oversized.bin", repeat('x', MaxUnpackedBytes + 1))], true) + let + local = scripts[0].find("PK\x03\x04", 4) + central = scripts[0].find("PK\x01\x02") + resourceCentral = scripts[0].find("PK\x01\x02", central + 4) + doAssert local >= 0 and central >= 0 and resourceCentral >= 0 + for offset in [local + 22, resourceCentral + 24]: + for byte in 0 ..< 4: + scripts[0][offset + byte] = '\0' + episode("gota", 10, scripts, failure = true, + expectedRuntimeError = "Uncompressed data exceeds output limit") + echo "GOTA aggregate and false-size ZIP limits report player failure" + scripts[0] = zipFixture([("policy.bas", repeat("' Oversized source for VM compilation.\n", 2000))], true) episode("gota", 10, scripts, failure = true, diff --git a/docs/neural-runners.md b/docs/neural-runners.md index f50ade28..6fbb676e 100644 --- a/docs/neural-runners.md +++ b/docs/neural-runners.md @@ -73,14 +73,18 @@ No common neural manifest is required. Architectures define their own binary formats, and unused auxiliary files remain read-only resources. Zippy reads the archive and returns the BASIC entry and binary resources as -strings. Uploads are limited to 16 MiB and packages to 256 files. The loader +strings. Uploads are limited to 16 MiB and packages to 256 files. Decoded +source, resource contents, resource names and accounting overhead share a +64 MiB package budget. Each extraction receives the remaining budget and +rejects excess output during decompression, even with false ZIP size fields. +This bounds retained package data, not total process memory. The loader requires one BASIC file and canonical resource paths. ZIP decoding, compression support and checksum checks belong to Zippy, including stored, deflated and ZIP64 archives. Polyworld does not implement ZIP parsing or decompression. After unpacking, the VM compiler applies its existing BASIC source limit. -Native model and buffer limits apply when the neural runner is used. There is -no separate expanded-size limit in the ZIP decoder. The current Zippy reader +Native model and buffer limits apply when the neural runner is used. +The current Zippy reader requires a filename, so byte uploads temporarily stage the compressed archive; entry contents stay in memory, and the temporary ZIP is removed after reading. Errors use `PolicyError`, `BasicError` or `NeuralError` and reach the existing diff --git a/nimby.lock b/nimby.lock index 808097e1..0ebdc672 100644 --- a/nimby.lock +++ b/nimby.lock @@ -6,7 +6,7 @@ silky 0.2.0 https://github.com/treeform/silky 3c5dd562924c044bfd4417331f82132fa0 pixie 6.1.0 https://github.com/treeform/pixie 30bb510877f02cbf9643e4f904e33d1c90f7910f vmath 3.0.0 https://github.com/treeform/vmath bc260143102153b76328783e1c76c96e52c52a4b chroma 1.0.0 https://github.com/treeform/chroma 08b0d65590221bb964544fd9badf29a146b9091a -zippy 0.10.20 https://github.com/guzba/zippy e41dfa36a19d739a2be7493211decadbe451031d +zippy 0.10.20 https://github.com/relh/zippy d605465c98b9666f9a5232cece5a0441c10d159e flatty 0.4.0 https://github.com/treeform/flatty 9c3f14cc287890b742196b4f0f74f0b9faab29cb nimsimd 1.3.2 https://github.com/guzba/nimsimd dd27017fbfd1c6c00d37776a8f5d322bf6c5a83b bumpy 1.1.3 https://github.com/treeform/bumpy 93583f6f8056a6259aaa96c89f4c143ba55bc6f0 diff --git a/src/polyworld/policies.nim b/src/polyworld/policies.nim index 74409d80..3b0610f6 100644 --- a/src/polyworld/policies.nim +++ b/src/polyworld/policies.nim @@ -5,6 +5,7 @@ import const MaxPackageBytes* = 16 * 1024 * 1024 MaxPackageEntries* = 256 + MaxUnpackedBytes* = 64 * 1024 * 1024 type PolicyError* = object of CatchableError @@ -54,11 +55,14 @@ proc unpackPolicy(path: string): Policy = program = name if program.len == 0: fail("ZIP package must contain exactly one .bas file") - result.source = archive.extractFile(program) + result.source = archive.extractFile(program, MaxUnpackedBytes) result.memoryBytes = int64(result.source.len) for name in names: if name != program: - let bytes = archive.extractFile(name) + let remaining = MaxUnpackedBytes - int(result.memoryBytes) - name.len - 128 + if remaining < 0: + fail("Unpacked policy exceeds 64 MiB") + let bytes = archive.extractFile(name, remaining) result.files.add PolicyFile(name: name, bytes: bytes) result.memoryBytes += int64(name.len) + int64(bytes.len) + 128