From e7649cd8aa2088d83d61da3d03a907a28df4169a Mon Sep 17 00:00:00 2001 From: dayfine Date: Wed, 4 Nov 2026 21:38:03 -0800 Subject: [PATCH 1/2] Implement pinned map consumer updates and integrity checks --- .github/workflows/tests.yml | 38 +++ docs/autostory-plan.md | 9 +- docs/maps/consumer-process.md | 24 +- docs/maps/runtime-maps.md | 5 +- integrations/arduino-source/README.md | 105 +++++++ .../map-pack-integrity.yml.example | 19 ++ .../verify-map-packs/action.yml | 22 ++ src/pokesim/maps/consumer_source.py | 145 ++++++++++ src/pokesim/maps/consumer_sync.py | 201 +++++++++++++ src/pokesim/maps/test_consumer_sync.py | 263 ++++++++++++++++++ 10 files changed, 814 insertions(+), 17 deletions(-) create mode 100644 integrations/arduino-source/README.md create mode 100644 integrations/arduino-source/map-pack-integrity.yml.example create mode 100644 integrations/arduino-source/verify-map-packs/action.yml create mode 100644 src/pokesim/maps/consumer_source.py create mode 100644 src/pokesim/maps/consumer_sync.py create mode 100644 src/pokesim/maps/test_consumer_sync.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7b94130..c0ccb57 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,42 @@ on: pull_request: jobs: + map-consumer: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Initialize a consumer fixture from approved published maps + env: + PYTHONPATH: src + run: >- + python3 -m pokesim.maps.consumer_sync update + --consumer "$GITHUB_WORKSPACE" --source "$GITHUB_WORKSPACE" --offline + --game bdsp --game frlg --revision "$(git rev-parse origin/main)" + - name: Exercise the Arduino integration action + uses: ./integrations/arduino-source/verify-map-packs + - name: Ensure a local map edit fails the consumer check + env: + PYTHONPATH: src + run: | + python3 - <<'PY' + from pathlib import Path + import subprocess + import sys + from pokesim.maps.consumer_sync import DESTINATIONS + target = next(Path(DESTINATIONS['bdsp']).glob('*.json')) + target.write_bytes(target.read_bytes() + b'\n') + result = subprocess.run([sys.executable, '-m', 'pokesim.maps.consumer_sync', + 'check', '--consumer', '.', '--source', '.pokesim-map-source', '--offline'], + capture_output=True, text=True) + assert result.returncode == 1 and 'modified ' + target.name in result.stderr, result + print('Consumer correctly rejected local drift: ' + target.name) + PY + unittest: runs-on: ubuntu-22.04 strategy: @@ -22,6 +58,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run the tests run: PYTHONPATH=src python -m unittest discover -v -s src -t src + - name: Audit published map inventory and hashes + run: PYTHONPATH=src python -m pokesim.maps.runtime_map_pack battle: runs-on: ubuntu-22.04 diff --git a/docs/autostory-plan.md b/docs/autostory-plan.md index 24a02ee..b0891fa 100644 --- a/docs/autostory-plan.md +++ b/docs/autostory-plan.md @@ -19,10 +19,11 @@ semantics match and keep game-specific adapters where they differ. pinned exports, remote-manifest hash verification and map corrections here. - [ ] Recover exact pret revisions for BDSP inputs recorded as default branch; archive/pin inputs before refreshing them. -- [ ] Implement the pinned consumer lock/update/integrity check, configure it - as required, and adopt it in the owning Arduino PRs. Remove duplicate offline - authoring there; preserve native controller/detector tests. Design is recorded; - implementation and consumer adoption remain pending. +- [x] Implement pinned consumer lock/update/check commands, a reusable CI action + and regression coverage. See [the integration](../integrations/arduino-source/README.md). +- [ ] Adopt the check in owning Arduino PRs and configure its required status. + Resolve pending map differences first, remove duplicate offline authoring + there and preserve native controller/detector tests. - [ ] Consolidate the older eight `pokenav-map/1` fixtures only after explicit frame conversion and parity checks; do not overwrite them with cropped maps. diff --git a/docs/maps/consumer-process.md b/docs/maps/consumer-process.md index 4c30337..3fb5716 100644 --- a/docs/maps/consumer-process.md +++ b/docs/maps/consumer-process.md @@ -1,7 +1,9 @@ # Map changes and consumer updates -Status: proposed process; implementation pending. This document does not -install an Arduino check, import PR #226, or mark the consumer migration complete. +Status: check/update tooling and a reusable CI action are implemented; adoption +in Arduino and its required status settings remain pending. See the +[integration commands and rollout](../../integrations/arduino-source/README.md). +No changes from PR #226 are imported, and consumer migration is not complete. PokemonSimulation is the authoring repository for map data, source parsers, builders, schemas and corrections. Arduino-Source consumes a pinned export. @@ -34,10 +36,10 @@ Audit dependent CLI wrappers before removing their decoder imports. Route drawing/planning tools are a separate migration; do not delete all of `devtools/maps` just because builders move. -Use one consumer lock file, provisionally +Use one consumer lock file, `devtools/maps/map-packs.lock.json`. Each selected game pins a full -PokemonSimulation commit, the manifest path and its SHA-256. This is a proposed -shape, with placeholders, not an existing working lock: +PokemonSimulation commit, the manifest path and its SHA-256. The update command +writes this shape; this example uses placeholders rather than a real lock: ```json { @@ -160,13 +162,13 @@ generator/correction change and evidence. The required consumer check would reject all five edited JSON files under the old pin, even if Arduino's local simulations passed. Changing the local lock hash alone would still fail against the manifest fetched from the pinned source. None of these changes is imported -by this design PR. +by this implementation. ## Rollout and remaining milestone 1 work -1. Implement this repo's publication checks and update tool contract. Existing - `runtime_map_pack` already validates bytes/inventory/links; it does not yet - publish a consumer lock or run reproducible source generation in CI. +1. Implemented: source audit in CI, pinned lock/check/update commands, + consumer CI action and regression tests. Reproducible source generation in + CI and consumer adoption remain pending. 2. Archive the exact BDSP source inputs used for current output with their file hashes. Recover pret commits where possible; if not recoverable, label the snapshot's original commit unresolved and pin its archived bytes. Do not @@ -184,5 +186,5 @@ by this design PR. namespace after frame/parity checks; carry their measurements with them. Do not overwrite them with runtime crops or let consumer exports include them. -Only then mark map ownership complete. This design completes the process -decision, not the implementation or adoption steps above. +Only then mark map ownership complete. The core tooling implements the process +decision; source-input pinning and Arduino adoption above remain open. diff --git a/docs/maps/runtime-maps.md b/docs/maps/runtime-maps.md index 5e0eee1..05f8559 100644 --- a/docs/maps/runtime-maps.md +++ b/docs/maps/runtime-maps.md @@ -5,8 +5,9 @@ and offline checks. Arduino remains the consumer of generated JSON. Consumer branches have not been changed or had their generators deleted by this import. The staged work is tracked in [the integration plan](../autostory-plan.md). The [consumer process](consumer-process.md) specifies pinned exports and hash -checks so future map corrections are authored here. It is a design awaiting -implementation; Arduino's map directories are not yet protected by that gate. +checks so future map corrections are authored here. The +[check/update tools and CI action](../../integrations/arduino-source/README.md) +are implemented; Arduino adoption and required status settings are pending. ## Inventory and provenance diff --git a/integrations/arduino-source/README.md b/integrations/arduino-source/README.md new file mode 100644 index 0000000..bd9b9f3 --- /dev/null +++ b/integrations/arduino-source/README.md @@ -0,0 +1,105 @@ +# Arduino map consumer integration + +The implementation lives in `pokesim.maps.consumer_sync` and +`pokesim.maps.consumer_source`. Python 3.8+ and Git suffice; no Python packages +or battle engine are needed. This folder supplies the CI action and workflow +template. Installing it in the owning Arduino branches is still pending; +the template is not an active Arduino workflow. + +## Run the tooling + +From a PokemonSimulation checkout with these modules, set `PYTHONPATH=src`. +`--source` names a Git checkout of `dayfine/PokemonSimulation`; Git object reads +use the selected commit, never uncommitted source files. `--consumer` names an +Arduino checkout or an empty directory for a trial export. + +```sh +# Read-only preflight before adopting a pin; select a full publishing SHA. +PYTHONPATH=src python3 -m pokesim.maps.consumer_sync compare \ + --source . --consumer /path/to/Arduino-Source \ + --game bdsp --revision FULL_PUBLISHING_COMMIT + +# Initialize matching/empty exports, or advance previously clean locked packs. +PYTHONPATH=src python3 -m pokesim.maps.consumer_sync update \ + --source . --consumer /path/to/Arduino-Source \ + --game bdsp --revision FULL_PUBLISHING_COMMIT + +# Check every locked game, without changing consumer files. +PYTHONPATH=src python3 -m pokesim.maps.consumer_sync check \ + --source . --consumer /path/to/Arduino-Source +``` + +Repeat `--game` to initialize both FRLG and BDSP together. All existing map +directories must be locked; removing a game from the lock does not exempt its +maps. Games can subsequently advance independently. + +The default command refreshes source `origin/main` before verifying ancestry. +`--offline` explicitly trusts the cached `origin/main` ref and objects; it +does not establish their freshness or authenticate a manually fabricated cache. +The origin URL is restricted to PokemonSimulation. Shallow caches cannot be +refreshed by this tool; use a full-history checkout. An unmerged publishing +commit, mutable revision, missing object or unsupported format fails. + +`check` fails on altered bytes, missing/extra maps, symlinks, lock/manifest +inconsistency, unlocked game directories and retired authoring modules. +`compare` diagnoses a proposed pin without creating a lock or changing exports; +it is not a substitute for the CI `check` command. + +`update` validates the published source and current exports first. It will +not overwrite a pending local correction, drop an unknown file, or delete a +duplicate generator. Initial adoption accepts only empty directories or +byte-identical copies. For an existing lock, current packs must pass integrity +before an update can replace them. Files are staged as complete game directories; +ordinary swap errors roll back, and the lock is replaced last. A process kill +during swaps can require recovery; integrity fails rather than reporting success +under an inconsistent lock. Restore exports from the previous locked revision +before retrying. Review and commit exports and lock together. + +## Activate in the owning consumer branches + +1. Review pending map differences in PokemonSimulation first. In particular, + `compare` against the #40 publishing baseline flags #226's five modified + BDSP maps; this implementation does not approve or import those corrections. + Do not erase them to obtain a passing check. +2. Remove the duplicate authoring modules in their owning Arduino PRs, after + migrating dependent tools/tests. For BDSP these are `bdsp_maps.py` and + `decomp.py`; for FRLG, `frlg_maps.py`. The legacy `devtools/maps/__main__.py` + imports `decomp` and exposes its CLI, so update that dependency before + removing the parser. Keep unrelated drawing helpers and native readers. +3. Use `update` to create `devtools/maps/map-packs.lock.json` and the exact + selected packs. Consumer destinations are fixed to the existing CMake map + paths. No C++ reader, embedding or build-network change is necessary. +4. Add these narrow rules to the consumer's `.gitattributes` before checking + out/exporting on Windows; verify the resulting bytes with `check`: + + ```gitattributes + SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Maps/*.json text eol=lf + SerialPrograms/Source/VideoGames/PokemonFRLG/Overworld/Maps/*.json text eol=lf + ``` + +5. Copy `map-pack-integrity.yml.example` to + `.github/workflows/map-pack-integrity.yml`, replacing + `IMPLEMENTATION_COMMIT` with the full merged PokemonSimulation commit that + contains this action. The workflow runs for drafts too. Default PR checkout + checks the merge result; the action obtains full approved publishing history + independently and compares it to the consumer lock. +6. Run native reader/story tests against the selected exports and configure + `map-pack-integrity` as a required check on the actual merge target branches. + Apply the same gate to owning map PRs and their stacked story branches; + merely adding a check on fork main does not protect another stack's target. + Record this ownership policy in the Arduino agents instructions. Do not + claim enforcement until the workflow and required status are both active. + +The action runs the verifier from the pinned action commit and reads map data +from the independently checked-out publishing repository. Consumer-local +manifests or builder output are never its reference. Builds still use vendored +JSON and need no network access. + +## Validation of the implementation + +Colocated tests use real temporary Git repositories and cover drift in terrain, +points, NPCs, triggers, patches and metadata; inventory changes; forged local +manifest/hash; unmerged/mutable pins; symlinks; retired generators; independent +game updates; and rollback on a failed second directory swap. Simulation CI +also exercises the composite action with a complete real published export and +confirms that a subsequent local edit is rejected. diff --git a/integrations/arduino-source/map-pack-integrity.yml.example b/integrations/arduino-source/map-pack-integrity.yml.example new file mode 100644 index 0000000..e8f57f4 --- /dev/null +++ b/integrations/arduino-source/map-pack-integrity.yml.example @@ -0,0 +1,19 @@ +# Install as .github/workflows/map-pack-integrity.yml in the owning Arduino PR. +# Replace IMPLEMENTATION_COMMIT with this implementation's full merged commit. +name: Map pack integrity +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read +jobs: + map-pack-integrity: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: dayfine/PokemonSimulation/integrations/arduino-source/verify-map-packs@IMPLEMENTATION_COMMIT diff --git a/integrations/arduino-source/verify-map-packs/action.yml b/integrations/arduino-source/verify-map-packs/action.yml new file mode 100644 index 0000000..c0e2da7 --- /dev/null +++ b/integrations/arduino-source/verify-map-packs/action.yml @@ -0,0 +1,22 @@ +name: Verify PokemonSimulation map packs +description: Check vendored Arduino JSON against its pinned publishing commits. +runs: + using: composite + steps: + - name: Fetch approved map publishing history + uses: actions/checkout@v4 + with: + repository: dayfine/PokemonSimulation + ref: main + fetch-depth: 0 + path: .pokesim-map-source + persist-credentials: false + - name: Check locked map exports + shell: bash + env: + PYTHONPATH: ${{ github.action_path }}/../../../src + run: >- + python3 -m pokesim.maps.consumer_sync check + --consumer "$GITHUB_WORKSPACE" + --source "$GITHUB_WORKSPACE/.pokesim-map-source" + --offline diff --git a/src/pokesim/maps/consumer_source.py b/src/pokesim/maps/consumer_source.py new file mode 100644 index 0000000..6afc5f4 --- /dev/null +++ b/src/pokesim/maps/consumer_source.py @@ -0,0 +1,145 @@ +"""Read a published map pack from immutable Git objects, never checkout files.""" + +import hashlib +import json +from pathlib import Path +import re +import subprocess + +from .runtime_map_schema import validate + + +REPOSITORY = 'dayfine/PokemonSimulation' +MANIFEST = 'data/maps/runtime-manifest.json' +SHA = re.compile(r'[0-9a-f]{40}\Z') +DIGEST = re.compile(r'[0-9a-f]{64}\Z') +NAME = re.compile(r'[A-Za-z0-9_]+\Z') +GAMES = ('bdsp', 'frlg') + + +def sha256(raw): + return hashlib.sha256(raw).hexdigest() + + +def read_json(raw): + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate JSON key: ' + key) + result[key] = value + return result + return json.loads(raw, object_pairs_hook=unique) + + +class GitSource: + """A trusted cache of the publishing repository, with an approved main tip. + + CLI callers refresh by default. Offline use explicitly trusts the cache's + origin/main ref; it cannot establish freshness or remote authenticity. + """ + + def __init__(self, root): + self.root = Path(root).resolve() + origin = self.git('remote', 'get-url', 'origin').decode().strip() + if origin not in (f'https://github.com/{REPOSITORY}.git', + f'https://github.com/{REPOSITORY}', + f'git@github.com:{REPOSITORY}.git'): + raise ValueError('source origin must be ' + REPOSITORY) + + def git(self, *args): + result = subprocess.run(['git', '-C', str(self.root), *args], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode: + raise ValueError('source git: ' + result.stderr.decode(errors='replace').strip()) + return result.stdout + + def refresh(self): + if self.git('rev-parse', '--is-shallow-repository').strip() == b'true': + raise ValueError('source cache is shallow; fetch full history before verifying ancestry') + self.git('fetch', '--no-tags', 'origin', '+refs/heads/main:refs/remotes/origin/main') + + def approved(self, revision): + if not isinstance(revision, str) or not SHA.fullmatch(revision): + raise ValueError('revision must be a full lowercase commit SHA') + if self.git('cat-file', '-t', revision).strip() != b'commit': + raise ValueError('revision is not a commit') + try: + self.git('merge-base', '--is-ancestor', revision, 'refs/remotes/origin/main') + except ValueError: + raise ValueError('revision is not on cached origin/main (or ancestry is unavailable): ' + revision) + + def files(self, revision, directory): + """Map relative paths to blob IDs; reject symlinks and non-file entries.""" + entries = self.git('ls-tree', '-rz', revision, '--', directory).split(b'\0') + result = {} + for entry in entries: + if not entry: + continue + metadata, path = entry.split(b'\t', 1) + mode, kind, oid = metadata.decode().split() + if mode != '100644' or kind != 'blob': + raise ValueError('source must contain regular files: ' + path.decode()) + result[path.decode()] = oid + return result + + def read(self, revision, path): + entries = self.files(revision, path) + if path not in entries or len(entries) != 1: + raise ValueError('missing source file: ' + path) + return self.git('cat-file', 'blob', entries[path]) + + +def lock_entry(source, revision): + source.approved(revision) + return dict(revision=revision, manifest_path=MANIFEST, + manifest_sha256=sha256(source.read(revision, MANIFEST))) + + +def published_pack(source, game, pin): + """Return verified filename -> bytes for the selected, approved game pack.""" + if game not in GAMES: + raise ValueError('unknown game: ' + game) + if not isinstance(pin, dict) or set(pin) != {'revision', 'manifest_path', 'manifest_sha256'}: + raise ValueError('invalid pin fields: ' + game) + if pin['manifest_path'] != MANIFEST or not isinstance(pin['manifest_sha256'], str) or not DIGEST.fullmatch(pin['manifest_sha256']): + raise ValueError('invalid manifest path/hash: ' + game) + revision = pin['revision'] + source.approved(revision) + raw = source.read(revision, MANIFEST) + if sha256(raw) != pin['manifest_sha256']: + raise ValueError('published manifest hash mismatch: ' + game) + manifest = read_json(raw) + if not isinstance(manifest, dict) or manifest.get('format') != 'pokesim-map-pack/1' or manifest.get('runtime_format') != 'pokesim-runtime-map/1': + raise ValueError('unsupported map manifest/runtime format') + if not isinstance(manifest.get('games'), dict) or game not in manifest['games']: + raise ValueError('game not published: ' + game) + directory = f'data/games/{game}/maps/runtime' + entries = manifest['games'][game]['maps'] + if not isinstance(entries, list) or not entries: + raise ValueError('empty or invalid published pack: ' + game) + files = source.files(revision, directory) + result = {} + expected_paths = set() + for entry in entries: + if not isinstance(entry, dict): + raise ValueError('invalid published map entry') + name = entry.get('name') + if not isinstance(name, str) or not NAME.fullmatch(name) or name + '.json' in result: + raise ValueError('invalid/duplicate published map name: ' + repr(name)) + path = f'{directory}/{name}.json' + if entry.get('file') != path: + raise ValueError('unexpected published path: ' + repr(entry.get('file'))) + expected_paths.add(path) + if path not in files: + raise ValueError('missing published map: ' + path) + content = source.git('cat-file', 'blob', files[path]) + if sha256(content) != entry.get('sha256'): + raise ValueError('published map hash mismatch: ' + path) + data = validate(read_json(content)) + if data['name'] != name or data['source'] != entry.get('source'): + raise ValueError('published map identity/source mismatch: ' + path) + result[name + '.json'] = content + if set(files) != expected_paths: + raise ValueError('published inventory differs from manifest: ' + game) + return result diff --git a/src/pokesim/maps/consumer_sync.py b/src/pokesim/maps/consumer_sync.py new file mode 100644 index 0000000..78e793e --- /dev/null +++ b/src/pokesim/maps/consumer_sync.py @@ -0,0 +1,201 @@ +"""Check or update Arduino's pinned map exports. Check never modifies files.""" + +import argparse +from contextlib import ExitStack +import json +from pathlib import Path +import shutil +import tempfile + +from .consumer_source import GAMES, REPOSITORY, GitSource, lock_entry, published_pack, read_json + + +LOCK = 'devtools/maps/map-packs.lock.json' +DESTINATIONS = { + 'bdsp': 'SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Maps', + 'frlg': 'SerialPrograms/Source/VideoGames/PokemonFRLG/Overworld/Maps', +} +RETIRED = { + 'bdsp': ('devtools/maps/bdsp_maps.py', 'devtools/maps/decomp.py'), + 'frlg': ('devtools/maps/frlg_maps.py',), +} + + +def safe_path(root, relative): + root = Path(root).resolve() + path = root + for part in Path(relative).parts: + if part in ('..', '.') or Path(part).is_absolute(): + raise ValueError('invalid consumer path: ' + relative) + path = path / part + if path.is_symlink(): + raise ValueError('symlink in consumer path: ' + relative) + return path + + +def read_lock(root): + path = safe_path(root, LOCK) + if not path.is_file(): + raise ValueError('missing consumer lock: ' + LOCK) + lock = read_json(path.read_bytes()) + if not isinstance(lock, dict) or set(lock) != {'format', 'repository', 'games'}: + raise ValueError('invalid lock fields') + if lock['format'] != 'pokesim-map-lock/1' or lock['repository'] != REPOSITORY: + raise ValueError('unsupported lock format/repository') + if not isinstance(lock['games'], dict) or not lock['games'] or not set(lock['games']) <= set(GAMES): + raise ValueError('invalid/empty lock games') + return lock + + +def inventory(root, game): + directory = safe_path(root, DESTINATIONS[game]) + if not directory.exists(): + return {} + if not directory.is_dir(): + raise ValueError('map destination is not a directory: ' + str(directory)) + result = {} + for path in sorted(directory.iterdir()): + if path.is_symlink() or not path.is_file() or path.suffix != '.json': + raise ValueError('unexpected consumer map entry: ' + str(path)) + result[path.name] = path.read_bytes() + return result + + +def differences(root, game, expected): + actual = inventory(root, game) + result = [] + result += [f'{game}: missing {name}' for name in sorted(set(expected) - set(actual))] + result += [f'{game}: extra {name}' for name in sorted(set(actual) - set(expected))] + result += [f'{game}: modified {name}' for name in sorted(set(actual) & set(expected)) + if actual[name] != expected[name]] + return result + + +def ownership_errors(root, games): + return ['retired map authoring module: ' + path + for game in games for path in RETIRED[game] if safe_path(root, path).exists()] + + +def check(root, source): + lock = read_lock(root) + errors = ownership_errors(root, lock['games']) + for game in GAMES: + if game not in lock['games']: + if safe_path(root, DESTINATIONS[game]).exists(): + errors.append('unlocked consumer map directory: ' + game) + continue + pin = lock['games'][game] + expected = published_pack(source, game, pin) + errors.extend(differences(root, game, expected)) + if errors: + raise ValueError('\n'.join(errors)) + return lock + + +def replace_exports(root, packs, lock): + """Stage complete directories, swap them, then atomically replace the lock. + + Roll back swapped directories on an ordinary I/O failure. A process kill + between swaps still fails integrity under the old lock; no false success. + """ + with ExitStack() as stack: + staged = [] + for game, files in packs.items(): + destination = safe_path(root, DESTINATIONS[game]) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(stack.enter_context(tempfile.TemporaryDirectory( + prefix='.pokesim-maps-', dir=str(destination.parent)))) + new = temporary / 'new' + new.mkdir() + for name, raw in files.items(): + (new / name).write_bytes(raw) + staged.append((destination, new, temporary / 'old')) + lock_path = safe_path(root, LOCK) + lock_path.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(stack.enter_context(tempfile.TemporaryDirectory( + prefix='.pokesim-lock-', dir=str(lock_path.parent)))) + new_lock = temporary / 'lock.json' + new_lock.write_bytes((json.dumps(lock, indent=2) + '\n').encode('utf-8')) + swapped = [] + try: + for destination, new, old in staged: + if destination.exists(): + destination.rename(old) + swapped.append((destination, old)) + new.rename(destination) + new_lock.replace(lock_path) + except OSError: + for destination, old in reversed(swapped): + if destination.exists(): + shutil.rmtree(destination) + if old.exists(): + old.rename(destination) + raise + + +def update(root, source, games, revision): + """Update clean locked exports; initialize only empty or matching copies. + + Unrecorded map edits and retired generators are never silently removed. + Source validation completes before changing consumer files. + """ + if not games or not set(games) <= set(GAMES) or len(set(games)) != len(games): + raise ValueError('select each known game once') + lock_path = safe_path(root, LOCK) + if lock_path.exists(): + lock = check(root, source) + else: + lock = dict(format='pokesim-map-lock/1', repository=REPOSITORY, games={}) + pin = lock_entry(source, revision) + packs = {game: published_pack(source, game, pin) for game in games} + errors = ownership_errors(root, games) + for game in GAMES: + if game not in games and game not in lock['games'] and safe_path(root, DESTINATIONS[game]).exists(): + errors.append('unlocked consumer map directory: ' + game) + if game in games and game not in lock['games'] and inventory(root, game): + errors.extend(differences(root, game, packs[game])) + if errors: + raise ValueError('cannot adopt/update; review local differences first:\n' + '\n'.join(errors)) + for game in games: + lock['games'][game] = dict(pin) + replace_exports(root, packs, lock) + return lock + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=('check', 'update', 'compare')) + parser.add_argument('--consumer', type=Path, required=True) + parser.add_argument('--source', type=Path, required=True, help='PokemonSimulation Git checkout/cache') + parser.add_argument('--offline', action='store_true', help='trust cached origin/main; do not fetch') + parser.add_argument('--game', action='append', choices=GAMES) + parser.add_argument('--revision', help='full publishing commit, reachable from source origin/main') + args = parser.parse_args(argv) + if args.command == 'check' and (args.game or args.revision): + parser.error('check reads all games/revisions from the lock') + if args.command != 'check' and (not args.game or not args.revision): + parser.error('update/compare require --game and --revision') + try: + source = GitSource(args.source) + if not args.offline: + source.refresh() + if args.command == 'check': + lock = check(args.consumer, source) + elif args.command == 'update': + lock = update(args.consumer, source, args.game, args.revision) + else: + pin = lock_entry(source, args.revision) + errors = ownership_errors(args.consumer, args.game) + for game in args.game: + errors.extend(differences(args.consumer, game, published_pack(source, game, pin))) + if errors: + raise ValueError('\n'.join(errors)) + lock = dict(games={game: pin for game in args.game}) + for game, pin in lock['games'].items(): + print(f'{game}: verified {pin["revision"]} manifest {pin["manifest_sha256"]}') + except (ValueError, OSError, KeyError, TypeError) as error: + parser.exit(1, str(error) + '\n') + + +if __name__ == '__main__': + main() diff --git a/src/pokesim/maps/test_consumer_sync.py b/src/pokesim/maps/test_consumer_sync.py new file mode 100644 index 0000000..ad0e8a6 --- /dev/null +++ b/src/pokesim/maps/test_consumer_sync.py @@ -0,0 +1,263 @@ +"""Exercise the consumer boundary with actual Git history and temporary exports.""" + +import copy +from contextlib import redirect_stderr, redirect_stdout +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +from . import consumer_sync as sync +from .consumer_source import GitSource, MANIFEST, lock_entry, published_pack, sha256 + + +class ConsumerTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.publisher = self.root / 'publisher' + self.publisher.mkdir() + self.consumer = self.root / 'consumer' + self.consumer.mkdir() + self.git('init', '-q') + self.git('remote', 'add', 'origin', 'https://github.com/dayfine/PokemonSimulation.git') + self.maps = {} + for game in ('bdsp', 'frlg'): + self.maps[game] = {name: dict(name=name, source=dict(decomp_file='fixture', revision='fixed', date='test'), + tiles=['..', '..'], points={'door': [0, 0]}, + objects=[dict(name='npc', kind='npc', x=1, y=1)]) for name in ('A', 'B')} + self.base = self.publish() + self.source = GitSource(self.publisher) + + def git(self, *args): + return subprocess.check_output(['git', '-C', str(self.publisher), *args], stderr=subprocess.PIPE).decode().strip() + + def commit(self, approved=True): + self.git('add', '-A') + self.git('-c', 'user.name=Map test', '-c', 'user.email=maps@example.test', + 'commit', '--allow-empty', '-qm', 'map fixture') + revision = self.git('rev-parse', 'HEAD') + if approved: + self.git('update-ref', 'refs/remotes/origin/main', revision) + return revision + + def publish(self, change_manifest=None, approved=True): + manifest = dict(format='pokesim-map-pack/1', runtime_format='pokesim-runtime-map/1', games={}) + for game, maps in self.maps.items(): + directory = self.publisher / f'data/games/{game}/maps/runtime' + directory.mkdir(parents=True, exist_ok=True) + for path in directory.glob('*.json'): + path.unlink() + entries = [] + for name, data in maps.items(): + path = directory / (name + '.json') + raw = (json.dumps(data) + '\n').encode() + path.write_bytes(raw) + entries.append(dict(name=name, file=path.relative_to(self.publisher).as_posix(), + sha256=sha256(raw), source=data['source'])) + manifest['games'][game] = dict(maps=entries) + if change_manifest: + change_manifest(manifest) + path = self.publisher / MANIFEST + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(manifest) + '\n') + return self.commit(approved) + + def initialize(self, games=('bdsp',)): + return sync.update(self.consumer, self.source, games, self.base) + + def map_path(self, name='A', game='bdsp'): + return self.consumer / sync.DESTINATIONS[game] / (name + '.json') + + def snapshot(self): + return {p.relative_to(self.consumer).as_posix(): p.read_bytes() + for p in self.consumer.rglob('*') if p.is_file()} + + def rewrite_lock(self, change): + path = self.consumer / sync.LOCK + lock = json.loads(path.read_text()) + change(lock) + path.write_text(json.dumps(lock)) + + def test_initialize_check_and_independent_game_update(self): + self.initialize(('bdsp', 'frlg')) + old_frlg = self.map_path(game='frlg').read_bytes() + del self.maps['bdsp']['A'] + self.maps['bdsp']['C'] = dict(self.maps['bdsp']['B'], name='C') + revision = self.publish() + sync.update(self.consumer, self.source, ['bdsp'], revision) + lock = sync.check(self.consumer, self.source) + self.assertEqual(lock['games']['bdsp']['revision'], revision) + self.assertEqual(lock['games']['frlg']['revision'], self.base) + self.assertFalse(self.map_path().exists()) + self.assertTrue(self.map_path('C').exists()) + self.assertEqual(self.map_path(game='frlg').read_bytes(), old_frlg) + + def test_changed_points_objects_triggers_and_metadata_are_drift(self): + self.initialize() + path = self.map_path() + original = path.read_bytes() + changes = [dict(tiles=['#.', '..']), dict(points={'door': [1, 0]}), + dict(objects=[]), dict(patches=[dict(x=0, y=0, glyph='#', measured_by='test')]), + dict(coord_events=[dict(x=0, y=0, width=1, length=1, var='x', value=0)]), + dict(source=dict(decomp_file='changed', revision='fixed', date='test'))] + for change in changes: + with self.subTest(change=change): + data = json.loads(original) + data.update(change) + path.write_text(json.dumps(data)) + before = self.snapshot() + with self.assertRaisesRegex(ValueError, 'modified A.json'): + sync.check(self.consumer, self.source) + self.assertEqual(self.snapshot(), before) + path.write_bytes(original) + sync.check(self.consumer, self.source) + + def test_missing_and_extra_inventory(self): + self.initialize() + self.map_path().unlink() + self.map_path('Unexpected').write_text('{}') + with self.assertRaisesRegex(ValueError, 'missing A.json\nbdsp: extra Unexpected.json'): + sync.check(self.consumer, self.source) + + def test_local_manifest_and_rehashed_lock_cannot_approve_edits(self): + self.initialize() + self.map_path().write_text('{}') + forged = self.consumer / MANIFEST + forged.parent.mkdir(parents=True) + forged.write_text('{"games": {"bdsp": {"maps": []}}}') + self.rewrite_lock(lambda lock: lock['games']['bdsp'].update(manifest_sha256=sha256(forged.read_bytes()))) + with self.assertRaisesRegex(ValueError, 'published manifest hash mismatch'): + sync.check(self.consumer, self.source) + + def test_checkout_edits_in_source_do_not_change_published_git_objects(self): + self.initialize() + (self.publisher / MANIFEST).write_text('broken local manifest') + (self.publisher / 'data/games/bdsp/maps/runtime/A.json').write_text('broken local map') + sync.check(self.consumer, self.source) + + def test_unknown_repo_mutable_revision_and_unmerged_commit_rejected(self): + self.initialize() + original = (self.consumer / sync.LOCK).read_bytes() + for change, message in [ + (lambda lock: lock.update(repository='other/repo'), 'repository'), + (lambda lock: lock['games']['bdsp'].update(revision='main'), 'full lowercase'), + ]: + with self.subTest(message=message): + (self.consumer / sync.LOCK).write_bytes(original) + self.rewrite_lock(change) + with self.assertRaisesRegex(ValueError, message): + sync.check(self.consumer, self.source) + candidate = self.publish(approved=False) + with self.assertRaisesRegex(ValueError, 'not on cached origin/main'): + lock_entry(self.source, candidate) + + def test_published_hash_schema_duplicate_name_and_path_errors(self): + for change in ('hash', 'format', 'duplicate', 'path', 'empty'): + with self.subTest(change=change): + def edit(manifest): + entries = manifest['games']['bdsp']['maps'] + if change == 'hash': entries[0]['sha256'] = '0' * 64 + if change == 'format': manifest['runtime_format'] = 'unsupported/99' + if change == 'duplicate': entries.append(copy.deepcopy(entries[0])) + if change == 'path': entries[0]['file'] = '../elsewhere.json' + if change == 'empty': entries.clear() + revision = self.publish(edit) + with self.assertRaises(ValueError): + published_pack(self.source, 'bdsp', lock_entry(self.source, revision)) + + def test_extra_published_file_not_in_manifest_is_rejected(self): + (self.publisher / 'data/games/bdsp/maps/runtime/Extra.json').write_text('{}') + revision = self.commit() + with self.assertRaisesRegex(ValueError, 'published inventory'): + published_pack(self.source, 'bdsp', lock_entry(self.source, revision)) + + def test_new_game_cannot_be_hidden_by_removing_lock_entry(self): + self.initialize(('bdsp', 'frlg')) + self.rewrite_lock(lambda lock: lock['games'].pop('frlg')) + with self.assertRaisesRegex(ValueError, 'unlocked consumer map directory: frlg'): + sync.check(self.consumer, self.source) + + def test_retired_builder_reappearing_fails(self): + self.initialize() + path = self.consumer / sync.RETIRED['bdsp'][0] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('# duplicate builder') + with self.assertRaisesRegex(ValueError, 'retired map authoring module'): + sync.check(self.consumer, self.source) + + def test_symlink_never_follows_or_replaces_external_map(self): + self.initialize() + external = self.root / 'external.json' + external.write_text('do not touch') + self.map_path().unlink() + self.map_path().symlink_to(external) + with self.assertRaisesRegex(ValueError, 'unexpected consumer map entry'): + sync.update(self.consumer, self.source, ['bdsp'], self.base) + self.assertEqual(external.read_text(), 'do not touch') + + def test_update_refuses_local_edits_without_changing_any_files(self): + self.initialize() + self.map_path().write_text('pending local correction') + before = self.snapshot() + revision = self.publish() + with self.assertRaisesRegex(ValueError, 'modified A.json'): + sync.update(self.consumer, self.source, ['bdsp'], revision) + self.assertEqual(self.snapshot(), before) + + def test_first_adoption_refuses_existing_unreviewed_maps(self): + self.map_path().parent.mkdir(parents=True) + self.map_path().write_text('pending local correction') + before = self.snapshot() + with self.assertRaisesRegex(ValueError, 'review local differences first'): + self.initialize() + self.assertEqual(self.snapshot(), before) + + def test_failed_second_directory_swap_rolls_back_maps_and_lock(self): + self.initialize(('bdsp', 'frlg')) + before = self.snapshot() + self.maps['bdsp']['A']['tiles'] = ['#.', '..'] + self.maps['frlg']['A']['tiles'] = ['.#', '..'] + revision = self.publish() + rename = Path.rename + destination = self.map_path(game='frlg').parent.resolve() + def fail_second(path, target): + if path.name == 'new' and target == destination: + raise OSError('simulated disk error') + return rename(path, target) + with mock.patch.object(Path, 'rename', fail_second), self.assertRaisesRegex(OSError, 'disk error'): + sync.update(self.consumer, self.source, ['bdsp', 'frlg'], revision) + self.assertEqual(self.snapshot(), before) + sync.check(self.consumer, self.source) + + def test_duplicate_lock_keys_are_rejected(self): + self.initialize() + path = self.consumer / sync.LOCK + path.write_text('{"games": {}, "games": {}}') + with self.assertRaisesRegex(ValueError, 'duplicate JSON key'): + sync.check(self.consumer, self.source) + + def test_cli_refreshes_by_default_and_offline_failure_is_read_only(self): + self.initialize() + args = ['check', '--consumer', str(self.consumer), '--source', str(self.publisher)] + with mock.patch.object(GitSource, 'refresh') as refresh, redirect_stdout(io.StringIO()): + sync.main(args) + refresh.assert_called_once_with() + self.map_path().write_text('local edit') + before = self.snapshot() + errors = io.StringIO() + with mock.patch.object(GitSource, 'refresh') as refresh, redirect_stderr(errors): + with self.assertRaises(SystemExit) as result: + sync.main(args + ['--offline']) + self.assertEqual(result.exception.code, 1) + refresh.assert_not_called() + self.assertIn('modified A.json', errors.getvalue()) + self.assertEqual(self.snapshot(), before) + + +if __name__ == '__main__': + unittest.main() From 7d2c07723268c29ec2aacf3d3f460e1c95d3df8c Mon Sep 17 00:00:00 2001 From: dayfine Date: Wed, 4 Nov 2026 21:39:25 -0800 Subject: [PATCH 2/2] Run consumer action from its pinned implementation directory --- integrations/arduino-source/verify-map-packs/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/arduino-source/verify-map-packs/action.yml b/integrations/arduino-source/verify-map-packs/action.yml index c0e2da7..7915330 100644 --- a/integrations/arduino-source/verify-map-packs/action.yml +++ b/integrations/arduino-source/verify-map-packs/action.yml @@ -13,6 +13,7 @@ runs: persist-credentials: false - name: Check locked map exports shell: bash + working-directory: ${{ github.action_path }}/../../.. env: PYTHONPATH: ${{ github.action_path }}/../../../src run: >-