From 7c5abe508cb76bdb61fd30c1a772f288fca705ec Mon Sep 17 00:00:00 2001 From: zhangwei Date: Fri, 17 Jul 2026 15:07:37 +0800 Subject: [PATCH 1/3] feat(ai-engineering): add multi-platform API Example workflow Add repository-local Contract, platform implementation, independent verification, and cross-platform acceptance tooling. Use an Agora repository profile for SDK version sources and keep coverage and release knowledge specific to the overseas repository. --- .../api-example-release-iteration/SKILL.md | 121 ++ AGENTS.md | 11 + .../ai-engineering/case-maintenance-matrix.md | 112 ++ docs/ai-engineering/knowledge-index.md | 226 +++ docs/ai-engineering/pilot-runs/README.md | 27 + docs/ai-engineering/release-iteration-gate.md | 160 ++ docs/ai-engineering/release-known-issues.md | 20 + docs/ai-engineering/repository-profile.json | 59 + docs/ai-engineering/role-routing.json | 36 + docs/ai-engineering/templates/README.md | 16 + .../acceptance-manifest-template.json | 273 +++ .../templates/release-dry-run-template.md | 131 ++ docs/ai-engineering/tools/README.md | 102 ++ .../tools/generate_case_backlog.py | 222 +++ .../tools/generate_case_backlog_test.py | 124 ++ .../tools/orchestrate_case_execution.py | 1586 +++++++++++++++++ .../tools/orchestrate_case_execution_test.py | 1138 ++++++++++++ .../tools/prepare_case_execution.py | 424 +++++ .../tools/prepare_case_execution_test.py | 212 +++ .../tools/validate_acceptance_manifest.py | 1212 +++++++++++++ .../validate_acceptance_manifest_test.py | 700 ++++++++ 21 files changed, 6912 insertions(+) create mode 100644 .agent/skills/api-example-release-iteration/SKILL.md create mode 100644 docs/ai-engineering/case-maintenance-matrix.md create mode 100644 docs/ai-engineering/knowledge-index.md create mode 100644 docs/ai-engineering/pilot-runs/README.md create mode 100644 docs/ai-engineering/release-iteration-gate.md create mode 100644 docs/ai-engineering/release-known-issues.md create mode 100644 docs/ai-engineering/repository-profile.json create mode 100644 docs/ai-engineering/role-routing.json create mode 100644 docs/ai-engineering/templates/README.md create mode 100644 docs/ai-engineering/templates/acceptance-manifest-template.json create mode 100644 docs/ai-engineering/templates/release-dry-run-template.md create mode 100644 docs/ai-engineering/tools/README.md create mode 100644 docs/ai-engineering/tools/generate_case_backlog.py create mode 100644 docs/ai-engineering/tools/generate_case_backlog_test.py create mode 100644 docs/ai-engineering/tools/orchestrate_case_execution.py create mode 100644 docs/ai-engineering/tools/orchestrate_case_execution_test.py create mode 100644 docs/ai-engineering/tools/prepare_case_execution.py create mode 100644 docs/ai-engineering/tools/prepare_case_execution_test.py create mode 100644 docs/ai-engineering/tools/validate_acceptance_manifest.py create mode 100644 docs/ai-engineering/tools/validate_acceptance_manifest_test.py diff --git a/.agent/skills/api-example-release-iteration/SKILL.md b/.agent/skills/api-example-release-iteration/SKILL.md new file mode 100644 index 000000000..ec7433f0e --- /dev/null +++ b/.agent/skills/api-example-release-iteration/SKILL.md @@ -0,0 +1,121 @@ +--- +name: api-example-release-iteration +description: Use when turning an RTC API Examples product requirement into aligned Android, iOS, macOS, and Windows changes with attributed implementation and independent repository acceptance +--- + +# API Example Release Iteration + +Use this skill for one product requirement across all official platform families. The Lead coordinates a shared Contract, attributed platform delivery, independent platform Verification, and final cross-platform acceptance. + +## Required Inputs + +Identify before editing: + +- Product scenario and key RTC SDK APIs. +- New case, existing case update, docs-only, or SDK-version work. +- Reference case for behavior and parity. +- Verification budget and available platform hosts. +- Target RTC SDK version. + +Android, iOS, macOS, and Windows are required by default. Contract may choose the appropriate full/audio/Compose/UIKit/Objective-C project for each platform; marking a platform not required needs an explicit waiver reason. + +## Source Route + +Read root `AGENTS.md`, the repository profile, this knowledge index and gate, then each platform/project `AGENTS.md`, `ARCHITECTURE.md`, and available `query-cases`, `upsert-case`, and `review-case` skills. Project skills may use either `.agent/skills/` or `.agents/skills/`. Read `release-known-issues.md` only for release, SDK version, CI, signing, packaging, license, or third-party asset work. + +## Agent Topology + +```text +Lead + Contract + Android Implementation -> Android Verification + iOS Implementation -> iOS Verification + macOS Implementation -> macOS Verification + Windows Implementation -> Windows Verification + Cross-platform acceptance +``` + +This is one Lead plus nine independent `codex exec` runs. They are replayable requirement-workflow runs, not parent-managed Codex subagent threads. Contract executes once. In a shared checkout, platform Implementation agents run one at a time so each repository delta belongs to one run; platform Verification agents run concurrently after their corresponding Implementation passes. + +| Role Type | Responsibility | Profile | +| --- | --- | --- | +| Contract | Shared behavior, APIs, non-goals, reference, platform targets and allowed files | `standard` | +| Implementation | Query and modify exactly one platform target | `deep` | +| Verification | Findings-first review, parity, UX, and strongest valid platform checks | `review` | + +Implementation cannot approve its own platform. Every accepted artifact needs a distinct `agent_id`, Codex run provenance, model/profile, and input snapshot hash. + +## Execution + +Initialize one requirement: + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py init \ + --matrix docs/ai-engineering/case-maintenance-matrix.md \ + --feature "" \ + --target-sdk-version "" \ + --run-dir /tmp/api-example-requirement +``` + +The default `docs/ai-engineering/repository-profile.json` supplies repository-specific SDK package names and version sources. Use `--repository-profile` only for a checked-in alternative profile. The execution package records its repository-relative path and SHA-256. + +For a new requirement not yet actionable in the matrix, also pass `--sdk-family ""` and repeat `--key-api ""` as needed. + +Dispatch the shared Contract: + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement \ + --phase contract \ + --model "" +``` + +Dispatch all platform agents by omitting `--platform`. Implementation runs are serialized in the shared checkout for delta attribution; Verification runs fan out concurrently: + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement \ + --phase implementation \ + --model "" + +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement \ + --phase verification \ + --model "" +``` + +Use `--platform android|ios|macos|windows` for a single platform and `--retry` to replace a prior `FAIL` or `BLOCKED` artifact. Retrying Implementation invalidates that platform's old Verification and carries all attempts into one cumulative net repository delta. Each run has an independent 900-second default timeout, JSONL command log, host provenance, and target-project working directory so nested `AGENTS.md` files load automatically. + +Assemble only after reviewing all required platform artifacts: + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py assemble \ + --run-dir /tmp/api-example-requirement \ + --matrix docs/ai-engineering/case-maintenance-matrix.md \ + --final-status "" \ + --cross-platform-result "" \ + --cross-platform-evidence "" +``` + +## Platform Rules + +- Contract selects the target project and allowed files for each platform. +- The dispatcher runs platform roles from that target project. In a shared checkout it refreshes the package/dependency snapshot, completes and reconciles one Implementation before starting the next, derives that role's cumulative net repository delta, and rejects changes outside Contract paths. +- Each Implementation agent follows only its platform/project instructions. +- Platform source files, dependencies, and build scripts are never shared across roots. +- Verification covers code review, reference parity, discoverability, UX, and valid checks on the current verification host. Executed PASS/FAIL commands must bind to the Codex JSONL command event and exit code, Contract working directory, and recorded host platform. +- Contract and Verification runs may create ignored build output but must not modify tracked or untracked repository content. +- A required platform `FAIL` or `BLOCKED` forces the whole requirement to `BLOCKED`. +- Final non-`BLOCKED` acceptance requires all required platform review, parity, and build results to pass, plus cross-platform acceptance `PASS`. + +## Host Mismatch + +On macOS, Windows Verification may perform repository-local static review only. It must not download Windows SDK archives, emulate Windows, cross-compile, or use a substitute compiler as build evidence. Keep Windows Verification and the requirement `BLOCKED` until the Verification role is retried on a real Windows host with MSBuild evidence. + +## Matrix And Release + +Each platform Implementation may propose zero or more matrix updates. `DONE` requires final `PASS`, platform parity/build `PASS`, and no skipped checks. Release data is limited to repository SDK-version consistency. Jenkins packaging, QA validation, artifact distribution, and external website publication are outside this workflow and must not be recorded as manifest gates. + +## Final Output + +Report the shared Contract, each platform's files and Verification result, declared platform differences, SDK-version checks, manifest validation, and final status. Do not describe the workflow as fully autonomous: the Lead still advances phases, handles retries, and owns final cross-platform judgment. diff --git a/AGENTS.md b/AGENTS.md index af39f90f7..3249c06d4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,17 @@ Each platform directory contains its own `AGENTS.md` with platform-specific rule | macOS | `macOS/AGENTS.md` | | Windows | `windows/AGENTS.md` | +## AI Engineering Workflow + +For AI-assisted version iteration, product-to-implementation routing, multi-agent acceptance, or automatic API example generation, use the repository workflow in `.agent/skills/api-example-release-iteration/SKILL.md`. + +Supporting documents: +- `docs/ai-engineering/knowledge-index.md` - repository knowledge map for agents. +- `docs/ai-engineering/release-iteration-gate.md` - product, architecture, review, test, and UX acceptance gates. +- `docs/ai-engineering/role-routing.json` - shared Contract and independent platform role types with logical Codex profiles. +- `docs/ai-engineering/repository-profile.json` - repository-specific SDK package names and version sources used by the shared tools. +- `docs/ai-engineering/release-known-issues.md` - repository release-risk reference. + ## Cross-Platform Rules 1. Never share source files, build scripts, or SDK dependencies between platforms. diff --git a/docs/ai-engineering/case-maintenance-matrix.md b/docs/ai-engineering/case-maintenance-matrix.md new file mode 100644 index 000000000..f32a3a89f --- /dev/null +++ b/docs/ai-engineering/case-maintenance-matrix.md @@ -0,0 +1,112 @@ +# API Examples Case Maintenance Matrix + +This matrix is the planning artifact for cross-platform API Example parity work. It does not replace platform `AGENTS.md`, project `ARCHITECTURE.md`, or project `.agent/skills/*` / `.agents/skills/*`; every row must be verified against the target project before implementation. + +## How To Use + +1. Start from the product request and identify the feature name, SDK family, and key SDK APIs. +2. Find the matching row in this matrix. +3. For each target platform unit, verify the path with the project `query-cases` skill when present. +4. Treat each required platform project as an independent delivery/verification unit inside one requirement-level acceptance manifest. +5. Update this matrix when a case is added, moved, renamed, or confirmed not applicable. + +## Status Values + +| Status | Meaning | +| --- | --- | +| `DONE(path)` | Inventory says the case is implemented and expected to be registered at the listed path. Verify before editing. For new or changed parity work, use `DONE` only after the required reference, review, and build gates pass. | +| `PARTIAL(note)` | Exists but parity, registration, docs, verification, runtime smoke, or API coverage needs review. | +| `MISSING` | Required for parity but no implementation is currently recorded here. | +| `N/A(reason)` | Not applicable because the SDK family, platform, or project variant does not support it. | +| `UNKNOWN` | Not inventoried yet; run the project case query before making decisions. | + +## Active Parity Acceptance Rule + +For new or changed parity work, do not move a cell to `DONE` from compilation alone. + +The platform execution unit must record: + +- Reference Contract: source implementation, UI resources, registration, SDK overloads, enum/option values, defaults, limits, and expected success signals. +- Parity Checklist: final implementation matches the reference contract; similar cases are used only for framework patterns. +- Build Result: the target project compile/build command passes, or the exact blocker is recorded. + +If runtime/device behavior is required but unavailable, use `PARTIAL(...)` and note the pending smoke check instead of `DONE(...)`. + +## Platform Units + +| Unit | Project | Primary Source | +| --- | --- | --- | +| Android full | `Android/APIExample/` | `Android/APIExample/ARCHITECTURE.md` | +| Android audio | `Android/APIExample-Audio/` | `Android/APIExample-Audio/ARCHITECTURE.md` | +| Android Compose | `Android/APIExample-Compose/` | `Android/APIExample-Compose/ARCHITECTURE.md` | +| iOS UIKit | `iOS/APIExample/` | `iOS/APIExample/ARCHITECTURE.md` | +| iOS SwiftUI | `iOS/APIExample-SwiftUI/` | `iOS/APIExample-SwiftUI/ARCHITECTURE.md` | +| iOS Objective-C | `iOS/APIExample-OC/` | `iOS/APIExample-OC/ARCHITECTURE.md` | +| iOS audio | `iOS/APIExample-Audio/` | `iOS/APIExample-Audio/ARCHITECTURE.md` | +| macOS | `macOS/` | `macOS/ARCHITECTURE.md` | +| Windows | `windows/` | `windows/ARCHITECTURE.md` | + +## Pilot Matrix + +Seeded from the overseas `dev/4.6.4` project `ARCHITECTURE.md` indexes and source tree. The inventory audit resolved 191 `DONE` candidates against their target project roots; paths are relative to the projects listed in Platform Units. + +| Feature | SDK Family | Key APIs | Android full | Android audio | Android Compose | iOS UIKit | iOS SwiftUI | iOS Objective-C | iOS audio | macOS | Windows | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Join channel video | Full RTC | `joinChannel`, `setupLocalVideo`, `setupRemoteVideo` | `DONE(basic/JoinChannelVideo.java)` | `N/A(audio SDK)` | `DONE(samples/JoinChannelVideo.kt)` | `DONE(Basic/JoinChannelVideo/)` | `DONE(Basic/JoinChannelVideo/)` | `DONE(Basic/JoinChannelVideo/)` | `N/A(audio SDK)` | `DONE(Basic/JoinChannelVideo/)` | `PARTIAL(Basic/JoinChannelVideoByToken)` | Windows records token variant only; non-token parity gap. | +| Join channel video with token | Full RTC | `joinChannel` with token, local and remote video setup | `DONE(basic/JoinChannelVideoByToken.java)` | `N/A(audio SDK)` | `DONE(samples/JoinChannelVideoToken.kt)` | `DONE(Basic/JoinChannelVideo(Token)/)` | `DONE(Basic/JoinChannelVideo(Token)/)` | `DONE(Basic/JoinChannelVideo(Token)/)` | `N/A(audio SDK)` | `DONE(Basic/JoinChannelVideo(Token)/)` | `DONE(Basic/JoinChannelVideoByToken/)` | Verify token-generation behavior per platform before changing auth flow. | +| Join channel audio | Full RTC or audio SDK | `joinChannel`, `setAudioProfile`, audio routing or volume APIs | `DONE(basic/JoinChannelAudio.java)` | `DONE(basic/JoinChannelAudio.java)` | `DONE(samples/JoinChannelAudio.kt)` | `DONE(Basic/JoinChannelAudio/)` | `DONE(Basic/JoinChannelAudio/)` | `DONE(Basic/JoinChannelAudio/)` | `DONE(Basic/JoinChannelAudio/)` | `DONE(Basic/JoinChannelAudio/)` | `MISSING` | Windows has no basic audio-only join case. | +| Live streaming role switch | Full RTC | `setClientRole`, `joinChannel`, streaming role configuration | `DONE(advanced/LiveStreaming.java)` | `N/A(video/live)` | `DONE(samples/LiveStreaming.kt)` | `DONE(Advanced/LiveStreaming/)` | `DONE(Advanced/LiveStreaming/)` | `DONE(Advanced/LiveStreaming/)` | `N/A(video/live)` | `DONE(Advanced/LiveStreaming/)` | `DONE(Basic/LiveBroadcasting/)` | Names differ by platform; keep product scenario wording stable. | +| RTMP or CDN streaming | Full RTC | `startRtmpStreamWithTranscoding`, `startRtmpStreamWithoutTranscoding`, `stopRtmpStream` | `DONE(advanced/RTMPStreaming.java)` | `N/A(video/live)` | `DONE(samples/RTMPStreaming.kt)` | `DONE(Advanced/RTMPStreaming/)` | `DONE(Advanced/RTMPStream/)` | `DONE(Advanced/RTMPStreaming/)` | `N/A(video/live)` | `DONE(Advanced/RTMPStreaming/)` | `DONE(Advanced/RTMPStream/)` | Windows also has `RTMPinject`; verify requested API before routing. | +| Media metadata | Full RTC | `registerMediaMetadataObserver` or platform metadata delegate APIs | `DONE(advanced/MediaMetadata.java)` | `N/A(audio SDK)` | `DONE(samples/MediaMetadata.kt)` | `DONE(Advanced/VideoMetadata/)` | `DONE(Advanced/VideoMetadata/)` | `DONE(Advanced/VideoMetadata/)` | `N/A(audio SDK)` | `MISSING` | `DONE(Advanced/Metadata/)` | macOS has no metadata case. iOS names this VideoMetadata. | +| Data stream messaging | Full RTC | `createDataStream`, `sendStreamMessage` | `DONE(advanced/SendDataStream.java)` | `MISSING` | `DONE(samples/SendDataStream.kt)` | `DONE(Advanced/CreateDataStream/)` | `DONE(Advanced/CreateDataStream/)` | `DONE(Advanced/CreateDataStream/)` | `MISSING` | `DONE(Advanced/CreateDataStream/)` | `MISSING` | Windows, Android audio, and iOS audio have no data stream case. | +| Custom audio source | Full RTC or audio SDK | `createCustomAudioTrack`, `setExternalAudioSource`, `pushExternalAudioFrame` | `DONE(advanced/customaudio/CustomAudioSource.java)` | `DONE(advanced/customaudio/CustomAudioSource.java)` | `DONE(samples/CustomAudioSource.kt)` | `DONE(Advanced/CustomPcmAudioSource/)` | `DONE(Advanced/CustomPCMAudioSource/)` | `DONE(Advanced/CustomPcmAudioSource/)` | `DONE(Advanced/CustomAudioSource/)` | `DONE(Advanced/CustomAudioSource/)` | `DONE(Advanced/CustomAudioCapture/)` | Platform APIs differ; compare demonstrated behavior, not only method names. | +| Custom audio render | Full RTC or audio SDK | `setExternalAudioSink`, pull playback audio frame APIs | `DONE(advanced/customaudio/CustomAudioRender.java)` | `DONE(advanced/customaudio/CustomAudioRender.java)` | `DONE(samples/CustomAudioRender.kt)` | `DONE(Advanced/CustomAudioRender/)` | `DONE(Advanced/CustomAudioRender/)` | `DONE(Advanced/CustomAudioRender/)` | `DONE(Advanced/CustomAudioRender/)` | `DONE(Advanced/CustomAudioRender/)` | `MISSING` | Windows has no custom audio render case. | +| Custom video source | Full RTC | `setExternalVideoSource`, `pushExternalVideoFrame`, custom video track APIs | `DONE(advanced/PushExternalVideoYUV.java)` | `N/A(audio SDK)` | `DONE(samples/CustomVideoSource.kt)` | `DONE(Advanced/CustomVideoSourcePush/)` | `MISSING` | `DONE(Advanced/CustomVideoSourcePush/)` | `N/A(audio SDK)` | `DONE(Advanced/CustomVideoSourcePush/)` | `DONE(Advanced/CustomVideoCapture/)` | SwiftUI has CustomVideoRender but no source-push case. Multi-source variants are separate rows. | +| Raw audio data | Full RTC or audio SDK | `registerAudioFrameObserver`, audio frame delegate APIs | `DONE(advanced/ProcessAudioRawData.java)` | `DONE(advanced/ProcessAudioRawData.java)` | `DONE(samples/OriginAudioData.kt)` | `DONE(Advanced/RawAudioData/)` | `DONE(Advanced/RawAudioData/)` | `DONE(Advanced/RawAudioData/)` | `DONE(Advanced/RawAudioData/)` | `DONE(Advanced/RawAudioData/)` | `DONE(Advanced/OriginalAudio/)` | Check callback threading in every platform review. | +| Raw video data | Full RTC | `registerVideoFrameObserver`, video frame delegate APIs | `DONE(advanced/ProcessRawData.java)` | `N/A(audio SDK)` | `DONE(samples/OriginVideoData.kt)` | `DONE(Advanced/RawVideoData/)` | `DONE(Advanced/RawVideoData/)` | `DONE(Advanced/RawVideoData/)` | `N/A(audio SDK)` | `DONE(Advanced/RawVideoData/)` | `DONE(Advanced/OriginalVideo/)` | Raw media combined samples are tracked separately. | +| Screen sharing | Full RTC | `startScreenCapture`, `updateScreenCaptureParameters`, `stopScreenCapture` | `DONE(advanced/ScreenSharing.java)` | `N/A(audio SDK)` | `DONE(samples/ScreenSharing.kt)` | `DONE(Advanced/ScreenShare/)` | `DONE(Advanced/ScreenShare/)` | `DONE(Advanced/ScreenShare/)` | `N/A(audio SDK)` | `DONE(Advanced/ScreenShare/)` | `DONE(Advanced/ScreenShare/)` | iOS requires ReplayKit extension; Android requires foreground service. | +| Media player | Full RTC | `createMediaPlayer`, `open`, `play`, publish/update channel options | `DONE(advanced/MediaPlayer.java)` | `MISSING` | `DONE(samples/MediaPlayer.kt)` | `DONE(Advanced/MediaPlayer/)` | `DONE(Advanced/MediaPlayer/)` | `DONE(Advanced/MediaPlayer/)` | `MISSING` | `DONE(Advanced/MediaPlayer/)` | `DONE(Advanced/MediaPlayer/)` | Android audio and iOS audio have no standalone media player case. | +| Spatial audio | Full RTC or audio SDK | spatial audio engine, remote position, media player spatialization APIs | `DONE(advanced/SpatialSound.java)` | `DONE(advanced/SpatialSound.java)` | `DONE(samples/SpatialSound.kt)` | `DONE(Advanced/SpatialAudio/)` | `DONE(Advanced/SpatialAudio/)` | `DONE(Advanced/SpatialAudio/)` | `DONE(Advanced/SpatialAudio/)` | `DONE(Advanced/SpatialAudio/)` | `DONE(Advanced/SpatialAudio/)` | iOS and Android architecture docs record different key APIs; verify scenario first. | +| Stream encryption | Full RTC | `enableEncryption`, encryption configuration APIs | `DONE(advanced/ChannelEncryption.java)` | `MISSING` | `DONE(samples/ChannelEncryption.kt)` | `DONE(Advanced/StreamEncryption/)` | `DONE(Advanced/StreamEncryption/)` | `DONE(Advanced/StreamEncryption/)` | `MISSING` | `DONE(Advanced/StreamEncryption/)` | `DONE(Advanced/CustomEncrypt/ + MediaEncrypt/)` | Windows has multiple encryption rows; route by requested API. Audio projects lack encryption cases. | +| Pre-call test | Full RTC or audio SDK | `startEchoTest`, `stopEchoTest`, last-mile probe or network test APIs | `DONE(advanced/PreCallTest.java)` | `DONE(advanced/PreCallTest.java)` | `DONE(samples/PreCallTest.kt)` | `DONE(Advanced/PrecallTest/)` | `DONE(Advanced/PrecallTest/)` | `MISSING` | `DONE(Advanced/PrecallTest/)` | `DONE(Advanced/PrecallTest/)` | `DONE(Advanced/PreCallTest/)` | iOS Objective-C has no pre-call test case. | +| Third-party beauty | Full RTC | extension enablement, third-party SDK initialization, video frame processing | `DONE(advanced/ThirdPartyBeauty.java)` | `N/A(audio SDK)` | `MISSING` | `DONE(Advanced/ThirdBeautify/)` | `MISSING` | `MISSING` | `N/A(audio SDK)` | `MISSING` | `MISSING` | Only Android full and iOS UIKit have third-party beauty. Release work must check license expiration. | +| Extension sample | Full RTC extension | `enableExtension`, extension property APIs, matching native headers | `DONE(advanced/SimpleExtension.java)` | `N/A(audio SDK)` | `PARTIAL(VideoProcessExtension covers enableExtension)` | `DONE(Advanced/SimpleFilter/)` | `MISSING` | `DONE(Advanced/SimpleFilter/)` | `N/A(audio SDK)` | `DONE(Advanced/SimpleFilter/)` | `MISSING` | During SDK bumps, verify Android extension include files and iOS/macOS native assets. | +| Audio mixing | Full RTC or audio SDK | `startAudioMixing`, `stopAudioMixing`, `adjustAudioMixingVolume`, effect APIs | `DONE(advanced/PlayAudioFiles.java)` | `DONE(advanced/PlayAudioFiles.java)` | `DONE(samples/PlayAudioFiles.kt)` | `DONE(Advanced/AudioMixing/)` | `DONE(Advanced/AudioMixing/)` | `DONE(Advanced/AudioMixing/)` | `DONE(Advanced/AudioMixing/)` | `DONE(Advanced/AudioMixing/)` | `DONE(Advanced/AudioMixing/)` | Full coverage across all projects. | +| Rhythm player | Full RTC or audio SDK | `startRhythmPlayer`, `stopRhythmPlayer` | `DONE(advanced/RhythmPlayer.java)` | `DONE(advanced/RhythmPlayer.java)` | `DONE(samples/RhythmPlayer.kt)` | `DONE(Advanced/RhythmPlayer/)` | `DONE(Advanced/RhythmPlayer/)` | `DONE(Advanced/RhythmPlayer/)` | `DONE(Advanced/RhythmPlayer/)` | `MISSING` | `MISSING` | macOS and Windows have no rhythm player case. | +| Media recorder | Full RTC | `createMediaRecorder`, `startRecording`, `stopRecording` | `DONE(advanced/MediaRecorder.java)` | `N/A(audio SDK)` | `DONE(samples/MediaRecorder.kt)` | `DONE(Basic/JoinChannelVideo(Recorder)/)` | `DONE(Basic/JoinChannelVideo(Recorder)/)` | `DONE(Basic/JoinChannelVideo(Recorder)/)` | `N/A(audio SDK)` | `DONE(Basic/JoinChannelVideo(Recorder)/)` | `DONE(Advanced/MediaRecorder/)` | iOS/macOS embed recorder in a join-channel variant; Android/Compose/Windows have standalone cases. | +| Local video transcoding | Full RTC | `startLocalVideoTranscoder`, `startCameraCapture`, compositing APIs | `DONE(advanced/LocalVideoTranscoding.java)` | `N/A(audio SDK)` | `DONE(samples/LocalVideoTranscoding.kt)` | `DONE(Advanced/LocalCompositeGraph/)` | `DONE(Advanced/LocalVideoTranscoding/)` | `DONE(Advanced/LocalCompositeGraph/)` | `N/A(audio SDK)` | `DONE(Advanced/LocalVideoTranscoding/)` | `DONE(Advanced/LocalVideoTranscoding/)` | iOS UIKit and OC name this LocalCompositeGraph; SwiftUI splits transcoding and composition. | +| Multi-channel join | Full RTC | `joinChannelEx`, `leaveChannelEx`, multi-channel connection APIs | `DONE(advanced/JoinMultipleChannel.java)` | `N/A(audio SDK)` | `DONE(samples/JoinMultiChannel.kt)` | `DONE(Advanced/JoinMultiChannel/)` | `DONE(Advanced/JoinMultiChannel/)` | `DONE(Advanced/JoinMultiChannel/)` | `N/A(audio SDK)` | `DONE(Advanced/JoinMultiChannel/)` | `DONE(Advanced/MultiChannel/)` | Full coverage across full-RTC projects. | +| Multipath | Full RTC | `enableMultipath`, `setMultipathConfig`, or `updateChannel` with multipath options | `DONE(advanced/Multipath.java)` | `N/A(audio SDK)` | `MISSING` | `DONE(Advanced/Multipath/)` | `DONE(Advanced/Multipath/)` | `DONE(Advanced/Multipath/)` | `N/A(audio SDK)` | `DONE(Advanced/Multipath/)` | `DONE(Advanced/Multipath/)` | Android Compose has no multipath case. | +| Content inspect | Full RTC | `enableContentInspect` | `DONE(advanced/ContentInspect.java)` | `N/A(audio SDK)` | `MISSING` | `DONE(Advanced/ContentInspect/)` | `DONE(Advanced/ContentInspect/)` | `DONE(Advanced/ContentInspect/)` | `N/A(audio SDK)` | `DONE(Advanced/ContentInspect/)` | `MISSING` | Android Compose and Windows have no content inspect case. | +| Channel media relay | Full RTC | `startOrUpdateChannelMediaRelay`, `stopChannelMediaRelay`, pause/resume APIs | `DONE(advanced/HostAcrossChannel.java)` | `N/A(audio SDK)` | `DONE(samples/HostAcrossChannel.kt)` | `DONE(Advanced/MediaChannelRelay/)` | `DONE(Advanced/MediaChannelRelay/)` | `DONE(Advanced/MediaChannelRelay/)` | `N/A(audio SDK)` | `DONE(Advanced/ChannelMediaRelay/)` | `DONE(Advanced/CrossChannel/)` | Full coverage across full-RTC projects. | +| Picture in picture | Full RTC | `enterPictureInPictureMode` or `AVPictureInPictureController` | `DONE(advanced/PictureInPicture.java)` | `N/A(audio SDK)` | `DONE(samples/PictureInPicture.kt)` | `DONE(Advanced/PictureInPicture/)` | `DONE(Advanced/PictureInPicture/)` | `DONE(Advanced/PictureInPicture/)` | `N/A(audio SDK)` | `MISSING` | `MISSING` | macOS and Windows have no PiP case. | + +## Confirmed Gaps + +These gaps are derived from the pilot matrix above and should be prioritized when parity work is requested. + +| Gap | Affected Units | Severity | +| --- | --- | --- | +| Basic audio-only join channel | Windows | High — missing foundational case | +| Media metadata | macOS | Medium — full-RTC platform gap | +| Data stream messaging | Android audio, iOS audio, Windows | Medium — audio and Windows gap | +| Custom audio render | Windows | Medium — full-RTC platform gap | +| Media player (standalone) | Android audio, iOS audio | Low — audio projects may not need standalone player | +| Stream encryption | Android audio, iOS audio | Low — audio projects may not need encryption | +| Pre-call test | iOS Objective-C | Medium — full-RTC variant gap | +| Third-party beauty | Android Compose, iOS SwiftUI, iOS OC, macOS, Windows | Low — third-party SDK and license overhead | +| Extension sample | iOS SwiftUI, Windows | Medium — extension enablement gap | +| Rhythm player | macOS, Windows | Low — audio utility gap | +| Multipath | Android Compose | Low — Compose variant gap | +| Content inspect | Android Compose, Windows | Low — moderation feature gap | +| Picture in picture | macOS, Windows | Low — platform feature gap | +| Custom video source (push) | iOS SwiftUI | Medium — SwiftUI variant gap | + +## Maintenance Rules + +- Do not change a `UNKNOWN` cell to `MISSING` until the target project `query-cases` skill or architecture file has been checked. +- Do not change a `PARTIAL` cell to `DONE` unless registration, discoverability, reference parity, review, and build evidence are confirmed for the active work. +- Prefer feature names based on user-visible scenario rather than platform-specific class names. +- Keep API names in the matrix broad enough to route work, then validate exact signatures in the target SDK version. +- Keep release-risk notes in this matrix short; detailed release checks belong in `docs/ai-engineering/release-known-issues.md`. +- When a new case is added to any project, update the matching row in the same edit or follow-up. diff --git a/docs/ai-engineering/knowledge-index.md b/docs/ai-engineering/knowledge-index.md new file mode 100644 index 000000000..9f393d275 --- /dev/null +++ b/docs/ai-engineering/knowledge-index.md @@ -0,0 +1,226 @@ +# API Examples AI Engineering Knowledge Index + +This index is the first file to read when an agent turns a product request into API Example changes. It links the existing repository knowledge into one route so the agent does not rediscover platform rules, case patterns, build commands, and known failure modes from scratch. + +## Source Priority + +Use sources in this order: + +1. Product request or issue text provided by the user. +2. `AGENTS.md` at the repository root for cross-platform boundaries. +3. Platform `AGENTS.md` for platform selection and red lines. +4. Project `AGENTS.md` for concrete commands and project-level skills. +5. Project `ARCHITECTURE.md` for case index, registration rules, and canonical file layout. +6. `docs/ai-engineering/repository-profile.json` for repository-specific SDK package names and version sources. +7. `docs/ai-engineering/case-maintenance-matrix.md` for cross-platform parity planning and known gaps. +8. Existing project `.agent/skills/*/SKILL.md` or `.agents/skills/*/SKILL.md` for case query, creation, and review procedures. +9. `docs/ai-engineering/release-known-issues.md` for release packaging and pipeline risks. +10. Repository hooks and platform build scripts for final verification. + +Do not use memory or sibling repositories as the source of truth when the current checkout has conflicting instructions. Treat external knowledge as a hint, then verify it against this repository. + +## Repository Boundary + +The repository has four independent platform families: + +| Platform | Root | Primary Rules | +| --- | --- | --- | +| Android | `Android/` | `Android/AGENTS.md` | +| iOS | `iOS/` | `iOS/AGENTS.md` | +| macOS | `macOS/` | `macOS/AGENTS.md` | +| Windows | `windows/` | `windows/AGENTS.md` | + +Never share source files, build scripts, SDK dependencies, or generated project metadata across platform roots unless a platform rule explicitly says to do so. + +## Project Selection + +| Request Type | Default Target | +| --- | --- | +| Android full RTC, video, screen sharing, beauty, extensions | `Android/APIExample/` | +| Android voice-only or audio-only SDK behavior | `Android/APIExample-Audio/` | +| Android Compose case or Compose parity work | `Android/APIExample-Compose/` | +| iOS UIKit Swift full RTC | `iOS/APIExample/` | +| iOS SwiftUI parity work | `iOS/APIExample-SwiftUI/` | +| iOS Objective-C parity work | `iOS/APIExample-OC/` | +| iOS audio-only SDK behavior | `iOS/APIExample-Audio/` | +| macOS Swift Cocoa sample | `macOS/` | +| Windows C++ MFC sample | `windows/` | + +If the product request does not name a platform, default to Android, iOS, macOS, and Windows. Narrow the scope only when the user or Contract records an explicit waiver reason. Treat each required platform as a separate implementation and verification unit. + +## Existing Local Skills + +| Scope | Skills | +| --- | --- | +| `Android/APIExample/` | `query-cases`, `upsert-case`, `review-case` | +| `Android/APIExample-Audio/` | `query-cases`, `upsert-case`, `review-case` | +| `Android/APIExample-Compose/` | `query-cases`, `upsert-case`, `review-case` | +| `iOS/APIExample/` | `query-cases`, `upsert-case`, `review-case` | +| `iOS/APIExample-Audio/` | `query-cases`, `upsert-case`, `review-case` | +| `iOS/APIExample-OC/` | `query-cases`, `upsert-case`, `review-case` | +| `iOS/APIExample-SwiftUI/` | `query-cases`, `upsert-case`, `review-case` | +| `macOS/` | `upsert-case`, `review-case` | +| `windows/` | `upsert-case`, `review-case` | + +The repository-level orchestration skill is `.agent/skills/api-example-release-iteration/SKILL.md`. + +## Repository Profile + +`docs/ai-engineering/repository-profile.json` contains the SDK version sources that differ between API Examples distributions. Shared Python tools parse source kinds such as Gradle properties, CocoaPods packages, and SDK archive names without embedding distribution package names in code. + +The orchestrator stores the profile path and SHA-256 in the execution package and input snapshots. Dispatch and assembly stop if the checked-in profile changes after initialization. Coverage state and historical knowledge do not belong in this profile; keep them in the matrix and durable knowledge documents. + +## Codex Role Routing + +`docs/ai-engineering/role-routing.json` defines three role types and logical profiles. Provider model names remain runtime inputs through `--model` or `CODEX_MODEL_STANDARD`, `CODEX_MODEL_DEEP`, and `CODEX_MODEL_REVIEW`. The orchestrator starts replayable independent `codex exec` sessions; do not describe them as parent-managed Codex subagents. + +The phase order is: + +1. `contract`: one shared Contract for product behavior, reference, cross-platform invariants, and platform targets. +2. `implementation`: Android, iOS, macOS, and Windows agents run independently after Contract passes. A shared checkout serializes them and reconciles each delta before the next run. +3. `verification`: independent Android, iOS, macOS, and Windows agents run concurrently after their matching implementation passes. + +Release checks are mandatory manifest data, not an agent phase. Every platform dispatch has its own timeout, hashed JSONL command log, run identity, host platform, input snapshot, and Contract-selected working directory. Use `--platform` for a focused retry; otherwise the workflow covers all four official roots. + +## Agent Acceptance Artifacts + +Use `docs/ai-engineering/templates/acceptance-manifest-template.json` when an iteration changes source, workflow gates, matrix state, SDK-version logic, or docs that affect agent behavior. + +Validate the filled manifest with: + +```bash +python3 docs/ai-engineering/tools/validate_acceptance_manifest.py +``` + +The filled manifest is normally an execution artifact, not a repository document. Commit it only when the user explicitly asks for an evidence snapshot or when it is selected as a curated pilot-run example. + +For case backfill work, generate candidate platform execution units from the matrix: + +```bash +python3 docs/ai-engineering/tools/generate_case_backlog.py +``` + +Use the generated priority to select the next product requirement, then deliver that requirement across all four official platform families. Contract selects the applicable project variant inside each platform. + +## Case Implementation Knowledge + +For cross-platform parity requests: + +- Start with `docs/ai-engineering/case-maintenance-matrix.md`. +- Verify every relevant matrix cell against the target project before editing. +- Treat `UNKNOWN` as "not checked yet", not as a confirmed gap. +- Treat each required platform as an independent delivery/verification unit inside one requirement-level acceptance manifest. + +Before adding a case: + +- Use the target project's `query-cases` skill when available. +- Read the target project's `ARCHITECTURE.md` case index. +- Confirm whether the case already exists by feature name and SDK API name. +- Confirm the target group, registration mechanism, display name, and index rules. +- Extract the reference contract for parity work before implementation. The closest existing target-project case may guide lifecycle, permissions, UI framework, and registration patterns, but product semantics must come from the source reference contract. + +While adding or modifying a case: + +- Follow the target project's `upsert-case` skill. +- Keep edits inside the selected project unless the project-level instructions say otherwise. +- Update `ARCHITECTURE.md` when the case list, path, or key APIs change. +- Keep sensitive configuration placeholders intact. Do not commit real App IDs, certificates, tokens, or credentials. + +After implementation: + +- Run the target project's `review-case` skill. +- Run the applicable build or static check from the project `AGENTS.md`. +- Record which checks passed, failed, or were skipped with a reason. PASS/FAIL commands must match a JSONL `command_execution` event and exit code. +- Fill and validate an acceptance manifest before claiming final acceptance. + +## Knowledge Maintenance Protocol + +When an iteration uncovers a reusable failure pattern, record it in the right durable document instead of leaving it only in a chat summary. + +Use this structure: + +- Source: review finding, CI failure, release blocker, customer report, or manual smoke. +- Impact platform/project. +- Symptom. +- Root cause. +- Guardrail or rule. +- Verification command or evidence. +- Updated date. + +Where to write it: + +- Cross-platform routing or implementation traps: `docs/ai-engineering/knowledge-index.md`. +- Release, signing, CI, packaging, license, or SDK-version risks: `docs/ai-engineering/release-known-issues.md`. +- Case coverage state: `docs/ai-engineering/case-maintenance-matrix.md`. +- Project-specific implementation traps: the target project `ARCHITECTURE.md`, `.agent/skills/*/SKILL.md`, or `.agents/skills/*/SKILL.md`. + +Do not duplicate the same rule in every file. Put the durable rule at the lowest scope that future agents must read. + +When an acceptance manifest includes `knowledge_updates`, at least one platform Implementation must list the durable document or skill change in `files_changed`. The validator enforces this so new failure patterns cannot remain only in the execution summary. + +## Cross-Cutting Red Lines + +- Every case owns its RTC engine lifecycle. +- Leave the channel before destroying or releasing the engine. +- SDK callbacks can arrive on background threads; dispatch UI updates to the platform main thread. +- Do not hardcode real credentials. +- Do not mix full SDK and audio-only SDK APIs. +- Do not move common sample code across platform roots to reduce duplication. +- Do not edit packaging or CI scripts unless the product request or failure requires it. + +## Verification Entrypoints + +| Area | Entrypoint | +| --- | --- | +| Sensitive info and commit-message hooks | `HOOKS-GUIDE.md`, `.git-hooks/`, `.pre-commit-config.yaml`, `.gitleaks.toml` | +| Android build/test | project `AGENTS.md`, project Gradle files | +| iOS build | project `AGENTS.md`, project `Podfile`, `.github/ci/build/build_ios*.sh` | +| macOS build | `macOS/AGENTS.md`, `macOS/Podfile`, `.github/ci/build/build_mac*.sh` | +| Windows build | `windows/AGENTS.md`, `.github/ci/build/build_windows.*` | +| CI/CD | `azure-pipelines.yml`, `.github/ci/`, `.github/workflows/`, `cicd/` | +| Release known issues | `docs/ai-engineering/release-known-issues.md` | + +Prefer the smallest check that validates the changed surface. For docs-only changes, use static validation and link checks instead of full platform builds. + +## Pilot Run Reports + +`docs/ai-engineering/pilot-runs/` contains curated workflow validation examples, not a log of every agent execution. + +Use these reports to understand how a gate was exercised at a point in time. Do not use them as the source of truth for current SDK versions, case coverage, CI status, or release readiness. Re-check the live repository and current CI before reusing any conclusion. + +Only add a new pilot-run report when it is intentionally selected as a representative workflow example or when the user explicitly asks for a repository evidence snapshot. Routine run results should stay in the response, PR discussion, CI artifact, or task tracker. + +Use `docs/ai-engineering/templates/release-dry-run-template.md` as the starting point when such a repository evidence snapshot is needed. + +## Known Failure Patterns To Guard Against + +| Pattern | Guardrail | +| --- | --- | +| Wrong platform touched during a focused fix | Re-read root and platform `AGENTS.md`; list intended files before editing. | +| Case added but not visible in app | Verify project registration mechanism and case index. | +| Android case ordering collision | Use `query-cases` and scan source annotations before choosing a sort index. | +| SDK callback updates UI directly | Run `review-case`; check every callback that touches UI. | +| Engine leak after leaving screen | Check leave-channel before destroy/release in the real screen-close path. | +| Audio-only project receives video API | Re-check selected project SDK type before implementation. | +| CI diagnosis starts in the wrong layer | Identify the first failing log line before changing scripts or signing settings. | +| Third-party beauty license expires silently | Track expiration and confirm renewal before release branches are packaged. | +| Android Extension SDK headers drift from SDK version | Check extension `include` files during SDK version bumps. | +| iOS/macOS certificates expire on build machines | Inspect or print certificate expiration during release preparation. | +| SDK dependency version is not bumped on release branch | Verify platform SDK version files against the release target. | +| Model-declared build result is not backed by command telemetry | Bind each executed verification command to the hashed Codex JSONL log and exit code, and accept build status only for recognized platform build tools. | +| Platform run misses nested instructions | Start `codex exec` in the Contract-selected target so root/platform/project `AGENTS.md` files load automatically. | +| External packaging metadata leaks into repository acceptance | Keep Jenkins jobs, artifact URLs, QA evidence, and website publication outside the acceptance manifest. | +| Windows packaging fails from path length or permissions | Run Windows script preflight after path or packaging changes. | + +## Output Contract + +Every completed AI-assisted iteration should end with an acceptance summary: + +- Product request interpreted. +- Target platform/project. +- Files changed. +- Skills or docs used. +- Verification run and results. +- Open risks or skipped checks. + +Use `docs/ai-engineering/release-iteration-gate.md` for the full gate. diff --git a/docs/ai-engineering/pilot-runs/README.md b/docs/ai-engineering/pilot-runs/README.md new file mode 100644 index 000000000..13253ac77 --- /dev/null +++ b/docs/ai-engineering/pilot-runs/README.md @@ -0,0 +1,27 @@ +# Pilot Run Reports + +This directory stores curated workflow validation examples for the API Examples AI engineering process. + +Pilot-run reports are historical evidence. They show how a gate or workflow behaved at a specific branch, commit, and date. They are not the source of truth for current SDK versions, case coverage, CI status, signing state, or release readiness. + +Use `docs/ai-engineering/templates/release-dry-run-template.md` when a new release dry-run evidence snapshot is explicitly needed. + +## What Belongs Here + +- First dry run for a new repository gate or workflow. +- Representative cross-platform rehearsal that future agents can learn from. +- Explicitly requested evidence snapshot for a release or workflow decision. + +## What Does Not Belong Here + +- Every routine agent execution result. +- Repeated acceptance summaries for normal feature work. +- CI logs or build output that already belongs in CI artifacts. +- Mutable release status that must be checked live. + +## Maintenance Rules + +- Use dated filenames. +- Keep reviewed reports immutable; create a new report for a new run. +- Include branch, commit, scope, commands, skipped checks, and final status. +- Re-check live files and CI before using an old report as evidence. diff --git a/docs/ai-engineering/release-iteration-gate.md b/docs/ai-engineering/release-iteration-gate.md new file mode 100644 index 000000000..30753ae37 --- /dev/null +++ b/docs/ai-engineering/release-iteration-gate.md @@ -0,0 +1,160 @@ +# API Examples Release Iteration Gate + +One RTC product requirement is accepted across Android, iOS, macOS, and Windows. The Lead coordinates shared scope and cross-platform closure; platform implementation and verification remain independently owned. + +## Topology + +| Agent | Count | Pass Evidence | +| --- | --- | --- | +| Contract | 1 | Shared behavior, APIs, non-goals, reference, platform targets and constraints | +| Platform Implementation | 4 | Query/upsert result, scoped files, matrix proposals | +| Platform Verification | 4 | Findings, parity, entry/UX, commands, build result, skipped reasons | +| Lead | 1 | Cross-platform differences, final status, manifest assembly | + +Contract runs once. In a shared checkout, Android/iOS/macOS/Windows Implementation runs are serialized so every repository delta is attributable to one agent. Each Verification depends on Contract and its matching Implementation, runs concurrently with other platform Verification roles, and cannot reuse the Implementation agent identity. + +## Intake Gate + +Before Contract starts, identify feature behavior, SDK family, key APIs, target SDK version, reference case, expected user flow, repository scope, and available target verification hosts. Confirm that the checked-in repository profile describes the current SDK package names and version sources. All four top-level platforms are required by default. Contract may choose the appropriate project variant per platform. + +## Contract Gate + +Contract passes only when: + +- Scenario, inputs, success signals, APIs, and non-goals are observable. +- A reference source and result are recorded when parity is required. +- Cross-platform invariant behavior is explicit. +- Android, iOS, macOS, and Windows each have `required`, `target_project`, constraints, and allowed files. +- Lifecycle, callback threading, registration, SDK family, and credential boundaries are explicit. + +Nearby cases guide framework patterns; product behavior comes from the shared Contract/reference. + +## Platform Implementation Gate + +Each platform Implementation passes only when it: + +- Reads that platform and target-project instructions. +- Runs from the Contract-selected target project so nested `AGENTS.md` instructions are loaded by Codex. +- Queries existing cases before editing. +- Uses the selected project skill and stays inside Contract's allowed paths. +- Completes registration, localization, lifecycle cleanup, and architecture index updates. +- Lets the orchestrator derive its real platform file delta and records matrix proposals. + +Platform source roots are independent, but Implementation runs sharing one checkout are serialized so cross-root edits cannot be credited to the wrong agent. Each run refreshes an input snapshot bound to the execution package and dependency artifacts; retries retain attempt provenance and produce one cumulative net delta. Separate workspaces may execute them concurrently only when each produces an independently attributable patch. + +## Platform Verification Gate + +Each Verification independently checks: + +- Contract/reference parity, SDK types/defaults, lifecycle, threading, errors, and cleanup. +- Registration, display name, inputs, feedback, layout, and audio/video affordances. +- The strongest valid build or static command on the current verification host. +- Each executed PASS/FAIL command is bound to the hashed Codex JSONL command event and exit code; build results use recognized platform build actions from the Contract working directory. +- Dispatch provenance records the host platform. Windows build `PASS` requires a Windows host. +- The Verification run does not modify tracked or untracked repository content. +- Concrete reasons for every skipped check. + +For non-`BLOCKED` acceptance, every required platform needs role status, review result, parity result, and build result `PASS`. Final `PASS` permits no skipped checks. + +## Host Mismatch + +Host mismatch never authorizes SDK downloads, emulation, cross-compilation, or substitute compilers as evidence. On macOS, Windows Verification may report static findings but must keep `build_result=BLOCKED` and role status `BLOCKED`. A retry on a real Windows host can replace that artifact using `--platform windows --retry`. + +## Cross-Platform Gate + +After platform Verification, the Lead records: + +- `result`: `PASS`, `FAIL`, or `BLOCKED`. +- Evidence that all required platform contracts and behavior were compared. +- Intentional platform differences and their product/API justification. + +Any required platform blocker or cross-platform result other than `PASS` forces final `BLOCKED`. + +## Release Checklist + +Release is mandatory manifest data, not an agent. `requirement.target_sdk_version` and `release.target_sdk_version` must match. The orchestrator refreshes Android, iOS, macOS, and Windows dependency versions from live repository files during assembly. Non-`BLOCKED` acceptance requires all SDK-version checks to pass and permits no skipped repository release checks. + +Jenkins packaging, QA validation, artifact URLs, and external website publication are downstream processes outside this repository workflow. A final `PASS` means the API Example source and repository checks are ready for that external handoff; the manifest does not represent packaging or QA completion. + +## Manifest V4 + +The single state tree is: + +```text +requirement +contract +platforms.android.implementation / verification +platforms.ios.implementation / verification +platforms.macos.implementation / verification +platforms.windows.implementation / verification +cross_platform_acceptance +release +knowledge_updates +final_status +``` + +Validate it with: + +```bash +python3 docs/ai-engineering/tools/validate_acceptance_manifest.py +``` + +The validator enforces platform completeness, unique agent/run provenance, Contract target boundaries, platform result closure, cross-platform closure, matrix rules, SDK-version checks, repository paths, evidence-file hashes, and knowledge writeback. + +## Orchestration + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py init \ + --matrix docs/ai-engineering/case-maintenance-matrix.md \ + --feature "" \ + --target-sdk-version "" \ + --run-dir /tmp/api-example-requirement + +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement --phase contract --model "" + +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement --phase implementation --model "" + +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement --phase verification --model "" +``` + +If the product requirement is not yet an actionable matrix row, initialize it directly with `--sdk-family` and one or more `--key-api` arguments. + +Omitting `--platform` dispatches all four platform roles. Implementation runs are serialized in the shared checkout and reconciled after each run; Verification runs execute concurrently. Each independent `codex exec` run has a timeout, host provenance, a target-project working directory, and a hashed JSONL log. The execution package and input snapshots bind the checked-in repository profile by path and SHA-256. A prior `FAIL`/`BLOCKED` platform artifact can be replaced with `--retry`. + +## Automation Boundary + +The tools prepare platform agents, serialize shared-checkout implementation, parallelize immutable verification, enforce dependencies/timeouts, validate v4, and gate matrix changes. The Lead still approves phase advancement, retries, intentional differences, and final status. + +## Final Summary + +```markdown +## Acceptance Summary + +Requirement: +- Scenario/APIs: +- Contract/reference: + +Platforms: +- Android: files, verification, build +- iOS: files, verification, build +- macOS: files, verification, build +- Windows: files, verification, build + +Cross-platform: +- Result/evidence: +- Intentional differences: + +Release: +- SDK-version checks: +- External handoff required: Jenkins packaging and QA + +Manifest: +- Path/validation: + +Final status: PASS | PASS WITH RISKS | BLOCKED +``` + +Routine manifests remain execution artifacts unless explicitly selected as repository evidence. diff --git a/docs/ai-engineering/release-known-issues.md b/docs/ai-engineering/release-known-issues.md new file mode 100644 index 000000000..2fc609d5c --- /dev/null +++ b/docs/ai-engineering/release-known-issues.md @@ -0,0 +1,20 @@ +# API Examples Release Known Issues + +This file records repository-specific risks that can affect API Example source acceptance. Keep Jenkins packaging, QA validation, artifact distribution, and website publication outside the acceptance manifest. + +## Repository Acceptance Risks + +| Area | Risk | Repository Gate | +| --- | --- | --- | +| SDK versions | Android, iOS, macOS, and Windows dependency declarations can drift across release branches. | Validate every source declared in `repository-profile.json` against the target SDK version. | +| Android extensions | Extension samples can retain headers or native assets from an older SDK. | Review extension `include` files and matching assets whenever the SDK dependency changes. | +| Third-party beauty | Sample licenses and bundled assets can expire or become incompatible. | Record the current license or asset limitation when the related example changes. | +| iOS/macOS signing | Local verification may not cover external package-signing configuration. | Treat signing and packaging as an external handoff; do not report repository acceptance as package completion. | +| Windows verification | macOS cannot provide MSBuild evidence. | Keep Windows build verification blocked until the role runs on a real Windows host. | + +## Maintenance Rules + +- Keep target SDK package names and version sources in `repository-profile.json`, not Python code. +- Add a reusable failure pattern to this file only when it is verified against this repository. +- Keep case coverage in `case-maintenance-matrix.md`; do not use this file as a second coverage tracker. +- Do not add Jenkins URLs, package artifact URLs, QA ownership, or website publication state to the acceptance manifest. diff --git a/docs/ai-engineering/repository-profile.json b/docs/ai-engineering/repository-profile.json new file mode 100644 index 000000000..009841cc0 --- /dev/null +++ b/docs/ai-engineering/repository-profile.json @@ -0,0 +1,59 @@ +{ + "version": 1, + "sdk_version_sources": { + "android": [ + { + "path": "Android/APIExample/gradle.properties", + "kind": "gradle-property", + "key": "rtc_sdk_version" + }, + { + "path": "Android/APIExample-Audio/gradle.properties", + "kind": "gradle-property", + "key": "rtc_sdk_version" + }, + { + "path": "Android/APIExample-Compose/gradle.properties", + "kind": "gradle-property", + "key": "rtc_sdk_version" + } + ], + "ios": [ + { + "path": "iOS/APIExample/Podfile", + "kind": "cocoapods", + "package": "AgoraRtcEngine_iOS" + }, + { + "path": "iOS/APIExample-Audio/Podfile", + "kind": "cocoapods", + "package": "AgoraAudio_iOS" + }, + { + "path": "iOS/APIExample-OC/Podfile", + "kind": "cocoapods", + "package": "AgoraRtcEngine_iOS" + }, + { + "path": "iOS/APIExample-SwiftUI/Podfile", + "kind": "cocoapods", + "package": "AgoraRtcEngine_iOS" + } + ], + "macos": [ + { + "path": "macOS/Podfile", + "kind": "cocoapods", + "package": "AgoraRtcEngine_macOS" + } + ], + "windows": [ + { + "path": "windows/APIExample/install.ps1", + "kind": "archive-name", + "prefix": "Agora_Native_SDK_for_Windows_v", + "suffix": "_FULL.zip" + } + ] + } +} diff --git a/docs/ai-engineering/role-routing.json b/docs/ai-engineering/role-routing.json new file mode 100644 index 000000000..8e9d67cf0 --- /dev/null +++ b/docs/ai-engineering/role-routing.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "profiles": { + "standard": { + "reasoning_effort": "medium" + }, + "deep": { + "reasoning_effort": "high" + }, + "review": { + "reasoning_effort": "high" + } + }, + "phases": [ + "contract", + "implementation", + "verification" + ], + "roles": { + "contract": { + "profile": "standard", + "phase": "contract", + "sandbox": "read-only" + }, + "implementation": { + "profile": "deep", + "phase": "implementation", + "sandbox": "workspace-write" + }, + "verification": { + "profile": "review", + "phase": "verification", + "sandbox": "workspace-write" + } + } +} diff --git a/docs/ai-engineering/templates/README.md b/docs/ai-engineering/templates/README.md new file mode 100644 index 000000000..54079c830 --- /dev/null +++ b/docs/ai-engineering/templates/README.md @@ -0,0 +1,16 @@ +# AI Engineering Templates + +This directory stores reusable templates for repository-level AI engineering workflows. + +Templates are not completed evidence. Before using a template, replace every placeholder with live repository data and current command output. + +Templates may intentionally fail validation until their placeholders are replaced. The version 4 manifest stores one shared Contract and one Implementation/Verification pair for each official platform; use the orchestrator to add model, run, host, prompt/artifact, input-snapshot, working-directory, repository-delta, repository-profile, and hashed command-log provenance. Release data is limited to the target SDK version and repository checks. + +Do not commit routine execution output from these templates unless the user explicitly asks for a repository evidence snapshot or the run is selected as a representative workflow validation example. + +## Templates + +| Template | Purpose | +| --- | --- | +| `acceptance-manifest-template.json` | Requirement-level Contract, four platform delivery units, cross-platform acceptance, release, and knowledge evidence. | +| `release-dry-run-template.md` | Human-readable release dry-run evidence snapshot. | diff --git a/docs/ai-engineering/templates/acceptance-manifest-template.json b/docs/ai-engineering/templates/acceptance-manifest-template.json new file mode 100644 index 000000000..9b78562c8 --- /dev/null +++ b/docs/ai-engineering/templates/acceptance-manifest-template.json @@ -0,0 +1,273 @@ +{ + "version": 4, + "final_status": "BLOCKED", + "requirement": { + "feature": "", + "sdk_family": "", + "key_apis": [""], + "target_sdk_version": "" + }, + "contract": { + "agent_id": "contract-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/contract.md", + "artifact": "role-artifacts/contract.json" + }, + "status": "BLOCKED", + "evidence": "Pending shared Contract.", + "summary": "Pending shared behavior, reference, and platform targets.", + "output": { + "scenario": "", + "key_apis": [""], + "non_goals": [], + "reference": { + "required": false, + "source_case": "", + "contract_result": "SKIPPED" + }, + "cross_platform_requirements": [""], + "platform_targets": { + "android": { + "required": true, + "target_project": "Android/APIExample/", + "key_constraints": [], + "files_allowed": ["Android/APIExample/"], + "waiver_reason": "" + }, + "ios": { + "required": true, + "target_project": "iOS/APIExample/", + "key_constraints": [], + "files_allowed": ["iOS/APIExample/"], + "waiver_reason": "" + }, + "macos": { + "required": true, + "target_project": "macOS/", + "key_constraints": [], + "files_allowed": ["macOS/"], + "waiver_reason": "" + }, + "windows": { + "required": true, + "target_project": "windows/", + "key_constraints": [], + "files_allowed": ["windows/"], + "waiver_reason": "" + } + } + } + }, + "platforms": { + "android": { + "implementation": { + "agent_id": "android-implementation-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/android-implementation.md", + "artifact": "role-artifacts/android-implementation.json" + }, + "status": "BLOCKED", + "evidence": "Pending Android implementation.", + "summary": "Pending Android changes.", + "output": { + "target_project": "Android/APIExample/", + "query_cases": "Pending.", + "upsert_case": "Pending.", + "files_changed": [], + "matrix_updates": [] + } + }, + "verification": { + "agent_id": "android-verification-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/android-verification.md", + "artifact": "role-artifacts/android-verification.json" + }, + "status": "BLOCKED", + "evidence": "Pending Android verification.", + "summary": "Pending Android review and build.", + "output": { + "result": "BLOCKED", + "findings": [], + "parity_result": "BLOCKED", + "entry_point": "", + "ux_notes": "", + "commands": [], + "build_result": "BLOCKED", + "skipped_checks": [] + } + } + }, + "ios": { + "implementation": { + "agent_id": "ios-implementation-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/ios-implementation.md", + "artifact": "role-artifacts/ios-implementation.json" + }, + "status": "BLOCKED", + "evidence": "Pending iOS implementation.", + "summary": "Pending iOS changes.", + "output": { + "target_project": "iOS/APIExample/", + "query_cases": "Pending.", + "upsert_case": "Pending.", + "files_changed": [], + "matrix_updates": [] + } + }, + "verification": { + "agent_id": "ios-verification-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/ios-verification.md", + "artifact": "role-artifacts/ios-verification.json" + }, + "status": "BLOCKED", + "evidence": "Pending iOS verification.", + "summary": "Pending iOS review and build.", + "output": { + "result": "BLOCKED", + "findings": [], + "parity_result": "BLOCKED", + "entry_point": "", + "ux_notes": "", + "commands": [], + "build_result": "BLOCKED", + "skipped_checks": [] + } + } + }, + "macos": { + "implementation": { + "agent_id": "macos-implementation-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/macos-implementation.md", + "artifact": "role-artifacts/macos-implementation.json" + }, + "status": "BLOCKED", + "evidence": "Pending macOS implementation.", + "summary": "Pending macOS changes.", + "output": { + "target_project": "macOS/", + "query_cases": "Pending.", + "upsert_case": "Pending.", + "files_changed": [], + "matrix_updates": [] + } + }, + "verification": { + "agent_id": "macos-verification-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/macos-verification.md", + "artifact": "role-artifacts/macos-verification.json" + }, + "status": "BLOCKED", + "evidence": "Pending macOS verification.", + "summary": "Pending macOS review and build.", + "output": { + "result": "BLOCKED", + "findings": [], + "parity_result": "BLOCKED", + "entry_point": "", + "ux_notes": "", + "commands": [], + "build_result": "BLOCKED", + "skipped_checks": [] + } + } + }, + "windows": { + "implementation": { + "agent_id": "windows-implementation-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/windows-implementation.md", + "artifact": "role-artifacts/windows-implementation.json" + }, + "status": "BLOCKED", + "evidence": "Pending Windows implementation.", + "summary": "Pending Windows changes.", + "output": { + "target_project": "windows/", + "query_cases": "Pending.", + "upsert_case": "Pending.", + "files_changed": [], + "matrix_updates": [] + } + }, + "verification": { + "agent_id": "windows-verification-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/windows-verification.md", + "artifact": "role-artifacts/windows-verification.json" + }, + "status": "BLOCKED", + "evidence": "Pending Windows verification.", + "summary": "Pending Windows review and MSBuild evidence.", + "output": { + "result": "BLOCKED", + "findings": [], + "parity_result": "BLOCKED", + "entry_point": "", + "ux_notes": "", + "commands": [], + "build_result": "BLOCKED", + "skipped_checks": [] + } + } + } + }, + "cross_platform_acceptance": { + "result": "BLOCKED", + "evidence": "Pending all required platform verification.", + "differences": [] + }, + "release": { + "required": true, + "target_sdk_version": "", + "checks": [ + { + "name": "sdk-version-android", + "result": "BLOCKED", + "expected_version": "", + "actual_versions": {}, + "evidence": "", + "reason": "Pending live Android SDK version check." + }, + { + "name": "sdk-version-ios", + "result": "BLOCKED", + "expected_version": "", + "actual_versions": {}, + "evidence": "", + "reason": "Pending live iOS SDK version check." + }, + { + "name": "sdk-version-macos", + "result": "BLOCKED", + "expected_version": "", + "actual_versions": {}, + "evidence": "", + "reason": "Pending live macOS SDK version check." + }, + { + "name": "sdk-version-windows", + "result": "BLOCKED", + "expected_version": "", + "actual_versions": {}, + "evidence": "", + "reason": "Pending live Windows SDK version check." + } + ], + "skipped_checks": [] + }, + "knowledge_updates": [] +} diff --git a/docs/ai-engineering/templates/release-dry-run-template.md b/docs/ai-engineering/templates/release-dry-run-template.md new file mode 100644 index 000000000..c8ca2b8fe --- /dev/null +++ b/docs/ai-engineering/templates/release-dry-run-template.md @@ -0,0 +1,131 @@ +# Release Dry Run Report Template + +Use this template only when a release dry run is intentionally selected as a repository-level evidence snapshot. Routine execution results should stay in the agent response, PR discussion, CI artifact, or task tracker. + +Do not fill this template with stale values from an older run. Re-check the live repository and target release version before completing every section. + +## Scope + +- Repository: +- Branch: +- Commit: +- Target release SDK version: +- Product scenario: +- Target platform roots: +- Verification budget: +- Non-goals: + +## Sources Used + +- `AGENTS.md` +- `.agent/skills/api-example-release-iteration/SKILL.md` +- `docs/ai-engineering/knowledge-index.md` +- `docs/ai-engineering/repository-profile.json` +- `docs/ai-engineering/release-iteration-gate.md` +- `docs/ai-engineering/release-known-issues.md` +- Platform `AGENTS.md` files: +- Platform SDK version files: +- Platform build scripts: + +## Findings + +### Product Gate + +Status: + +Evidence: + +### Architecture Gate + +Status: + +Evidence: + +### Repository Release Gate + +Status: + +| Gate | Result | Evidence | +| --- | --- | --- | +| SDK version consistency | | | +| Android Extension include freshness | | | +| Third-party beauty license validity | | | +| Repository build checks | | | + +### Static Version Snapshot + +| Platform Project | Version Source | Observed Version | +| --- | --- | --- | +| `Android/APIExample` | `gradle.properties` | | +| `Android/APIExample-Audio` | `gradle.properties` | | +| `Android/APIExample-Compose` | `gradle.properties` | | +| `iOS/APIExample` | `Podfile` | | +| `iOS/APIExample-Audio` | `Podfile` | | +| `iOS/APIExample-OC` | `Podfile` | | +| `iOS/APIExample-SwiftUI` | `Podfile` | | +| `macOS` | `Podfile` | | +| `windows/APIExample` | SDK package or install script | | + +### Script Checks + +Commands: + +```bash + +``` + +Result: + +Not covered: + +- + +### Existing Automation Found + +| Area | Evidence | +| --- | --- | +| SDK version validation | | +| Build script preflight | | +| Signing visibility | | +| Windows packaging preflight | | + +## Automation Gaps + +1. + +## Acceptance Summary + +Product: +- Scenario: +- Target: +- Non-goals: + +Architecture: +- Platform/project: +- Key constraints: + +Implementation: +- Files changed: +- Skills/docs used: + +Review: +- Result: +- Findings: + +Testing: +- Commands: +- Result: +- Skipped checks: + +Release: +- Required: +- Target SDK version: +- Checks: +- Skipped checks: +- External handoff: Jenkins packaging and QA + +UX: +- Entry point: +- Notes: + +Final status: PASS | PASS WITH RISKS | BLOCKED diff --git a/docs/ai-engineering/tools/README.md b/docs/ai-engineering/tools/README.md new file mode 100644 index 000000000..f08dcccd8 --- /dev/null +++ b/docs/ai-engineering/tools/README.md @@ -0,0 +1,102 @@ +# AI Engineering Tools + +`orchestrate_case_execution.py` turns one RTC requirement into a shared Contract plus attributed Android, iOS, macOS, and Windows delivery. It launches independent, replayable `codex exec` sessions rather than parent-managed Codex subagent threads. + +## Role Types + +| Phase | Agents | Profile | +| --- | --- | --- | +| `contract` | One shared Contract | `standard` | +| `implementation` | Four platform Implementation agents | `deep` | +| `verification` | Four independent platform Verification agents | `review` | + +Provider model names remain runtime inputs through `--model` or `CODEX_MODEL_STANDARD`, `CODEX_MODEL_DEEP`, and `CODEX_MODEL_REVIEW`. + +## Run A Requirement + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py init \ + --matrix docs/ai-engineering/case-maintenance-matrix.md \ + --feature "Join channel audio" \ + --target-sdk-version "4.6.4" \ + --run-dir /tmp/api-example-requirement +``` + +For a new feature outside the matrix backlog, add `--sdk-family "Full RTC" --key-api ""`; repeat `--key-api` for multiple APIs. + +The default `docs/ai-engineering/repository-profile.json` defines repository-specific SDK version sources. Shared tools support Gradle property, CocoaPods package, and SDK archive-name sources without hard-coding a distribution in Python. `init` accepts `--repository-profile` for a checked-in alternative and binds its path and SHA-256 into the execution package. + +Dispatch Contract, then the two platform phases: + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement --phase contract --model "" + +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement --phase implementation --model "" + +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement --phase verification --model "" +``` + +Omitting `--platform` covers Android, iOS, macOS, and Windows. Implementation runs are serialized in a shared checkout and reconciled before the next agent starts; Verification runs execute concurrently. Use `--platform windows` for one platform. Use `--retry` only to replace a prior `FAIL` or `BLOCKED` artifact; an Implementation retry automatically invalidates that platform's old Verification. + +Each run starts in the Contract-selected target project, writes independent stdout/stderr logs, records its host platform, and has a 900-second default timeout. Input snapshots bind the execution package, current repository state, routing config, repository profile, and dependency artifact hashes. This lets Codex load nested platform/project `AGENTS.md` files automatically. `--dry-run` resolves prompts, content-aware snapshots, models, working directories, and commands without starting Codex. + +Verification PASS/FAIL commands are accepted only when their exact command string and exit code exist in the hashed Codex JSONL log. Build commands must execute a real target-platform build action from the Contract working directory; Windows build `PASS` additionally requires a Windows host. Contract and Verification may create ignored build output but fail if they modify tracked or untracked repository content. Implementation `files_changed` is derived from the real per-run repository delta rather than trusted from model output. + +Assemble with an explicit cross-platform conclusion: + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py assemble \ + --run-dir /tmp/api-example-requirement \ + --matrix docs/ai-engineering/case-maintenance-matrix.md \ + --final-status "BLOCKED" \ + --cross-platform-result "BLOCKED" \ + --cross-platform-evidence "Windows-host verification pending" +``` + +Manifest v4 stores one shared Contract and one Implementation/Verification pair per official platform. Implementation retries preserve attempt history and expose one cumulative net delta whose file list must match the manifest. Matrix updates are applied only after structural and evidence-file validation and never for final `BLOCKED`. Assembly also refreshes all live platform SDK dependency versions. + +Assembly does not accept CI job URLs, package artifact URLs, or QA metadata. Jenkins packaging, QA validation, artifact distribution, and website publication are external handoff processes rather than repository acceptance gates. + +## macOS And Windows + +Windows Verification on macOS is static review only. It must not download Windows SDKs, emulate, cross-compile, or treat another compiler as MSBuild evidence. Keep the Windows artifact `BLOCKED`, then replace it from a real Windows host with: + +```bash +python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ + --run-dir /tmp/api-example-requirement \ + --phase verification \ + --platform windows \ + --retry \ + --model "" +``` + +## Supporting Tools + +- `generate_case_backlog.py`: reads `MISSING`/`PARTIAL` matrix cells and prioritizes requirement candidates. +- `prepare_case_execution.py`: creates a v4 requirement package without starting Codex. +- `validate_acceptance_manifest.py`: validates a filled v4 manifest. + +```bash +python3 docs/ai-engineering/tools/prepare_case_execution.py \ + --feature "Join channel audio" \ + --target-sdk-version "4.6.4" \ + --repository-profile docs/ai-engineering/repository-profile.json +python3 docs/ai-engineering/tools/validate_acceptance_manifest.py +``` + +The standalone validator resolves evidence paths relative to the manifest and recomputes prompt, role-artifact, input-snapshot, command-log, and repository-delta hashes. + +Routine manifests and dispatch logs stay outside the repository unless the user requests an evidence snapshot. + +## Tests + +```bash +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest \ + docs/ai-engineering/tools/validate_acceptance_manifest_test.py \ + docs/ai-engineering/tools/generate_case_backlog_test.py \ + docs/ai-engineering/tools/prepare_case_execution_test.py \ + docs/ai-engineering/tools/orchestrate_case_execution_test.py +``` diff --git a/docs/ai-engineering/tools/generate_case_backlog.py b/docs/ai-engineering/tools/generate_case_backlog.py new file mode 100644 index 000000000..b7aedeca6 --- /dev/null +++ b/docs/ai-engineering/tools/generate_case_backlog.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Generate platform execution units from the case-maintenance matrix.""" + +import argparse +import json +import re +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_MATRIX = REPO_ROOT / "docs/ai-engineering/case-maintenance-matrix.md" +NON_PLATFORM_COLUMNS = {"Feature", "SDK Family", "Key APIs", "Notes"} +PLATFORM_PROJECTS = { + "Android full": "Android/APIExample/", + "Android audio": "Android/APIExample-Audio/", + "Android Compose": "Android/APIExample-Compose/", + "iOS UIKit": "iOS/APIExample/", + "iOS SwiftUI": "iOS/APIExample-SwiftUI/", + "iOS Objective-C": "iOS/APIExample-OC/", + "iOS audio": "iOS/APIExample-Audio/", + "macOS": "macOS/", + "Windows": "windows/", +} +SEVERITY_PRIORITY = { + "High": 10, + "Medium": 20, + "Low": 30, + "Unspecified": 90, +} +STATUS_PRIORITY_OFFSET = { + "MISSING": 0, + "PARTIAL": 5, +} +CONFIRMED_GAP_ALIASES = { + "Basic audio-only join channel": ["Join channel audio"], +} + + +def split_markdown_row(line): + return [cell.strip() for cell in line.strip().strip("|").split("|")] + + +def strip_code(value): + value = value.strip() + if value.startswith("`") and value.endswith("`"): + return value[1:-1] + return value + + +def parse_status(cell): + status, detail = parse_matrix_cell(cell) + if status in {"MISSING", "PARTIAL"}: + return status, detail + return "", "" + + +def parse_matrix_cell(cell): + value = strip_code(cell) + if value in {"MISSING", "UNKNOWN"}: + return value, "" + match = re.fullmatch(r"(DONE|PARTIAL|N/A)\((.+)\)", value) + if match: + return match.group(1), match.group(2) + return "", "" + + +def parse_done_path(cell): + status, detail = parse_matrix_cell(cell) + return detail if status == "DONE" else "" + + +def parse_key_apis(cell): + apis = re.findall(r"`([^`]+)`", cell) + if apis: + return [api.strip() for api in apis] + return [part.strip() for part in cell.split(",") if part.strip()] + + +def find_matrix_table(lines): + for index, line in enumerate(lines): + if not line.startswith("|"): + continue + header = split_markdown_row(line) + if "Feature" in header and "Key APIs" in header: + rows = [] + for row_line in lines[index + 2 :]: + if not row_line.startswith("|"): + break + rows.append(split_markdown_row(row_line)) + return header, rows + raise ValueError("could not find pilot matrix table with Feature and Key APIs columns") + + +def find_confirmed_gap_table(lines): + for index, line in enumerate(lines): + if not line.startswith("|"): + continue + header = split_markdown_row(line) + if {"Gap", "Affected Units", "Severity"}.issubset(set(header)): + rows = [] + for row_line in lines[index + 2 :]: + if not row_line.startswith("|"): + break + rows.append(split_markdown_row(row_line)) + return header, rows + return [], [] + + +def parse_severity(value): + label = re.split(r"\s+[-—]\s+", value.strip(), maxsplit=1)[0].strip() + return label if label in SEVERITY_PRIORITY else "Unspecified" + + +def gap_names_for_lookup(gap_name): + return [gap_name, *CONFIRMED_GAP_ALIASES.get(gap_name, [])] + + +def build_gap_severity_map(lines): + header, rows = find_confirmed_gap_table(lines) + if not header: + return {} + gap_index = header.index("Gap") + affected_index = header.index("Affected Units") + severity_index = header.index("Severity") + mapping = {} + for row in rows: + if len(row) != len(header): + continue + severity = parse_severity(row[severity_index]) + affected_units = [unit.strip() for unit in row[affected_index].split(",") if unit.strip()] + for unit in affected_units: + for gap_name in gap_names_for_lookup(row[gap_index]): + mapping[(gap_name, unit)] = severity + return mapping + + +def find_reference_candidates(header, row, target_platform): + candidates = [] + for column_index, column_name in enumerate(header): + if column_name in NON_PLATFORM_COLUMNS or column_name == target_platform: + continue + if column_name not in PLATFORM_PROJECTS: + continue + done_path = parse_done_path(row[column_index]) + if not done_path: + continue + candidates.append( + { + "platform_unit": column_name, + "project": PLATFORM_PROJECTS[column_name], + "path": done_path, + } + ) + return candidates + + +def generate_execution_units(matrix_path): + lines = matrix_path.read_text(encoding="utf-8").splitlines() + header, rows = find_matrix_table(lines) + gap_severities = build_gap_severity_map(lines) + feature_index = header.index("Feature") + sdk_index = header.index("SDK Family") + key_api_index = header.index("Key APIs") + notes_index = header.index("Notes") if "Notes" in header else None + execution_units = [] + errors = [] + + for row in rows: + if len(row) != len(header): + errors.append(f"row has {len(row)} cells but header has {len(header)} cells: {row}") + continue + for column_index, column_name in enumerate(header): + if column_name in NON_PLATFORM_COLUMNS: + continue + status, status_note = parse_status(row[column_index]) + if not status: + continue + if column_name not in PLATFORM_PROJECTS: + errors.append(f"unknown platform column with actionable status: {column_name}") + continue + severity = gap_severities.get((row[feature_index], column_name), "Unspecified") + priority = SEVERITY_PRIORITY[severity] + STATUS_PRIORITY_OFFSET[status] + execution_units.append( + { + "feature": row[feature_index], + "sdk_family": row[sdk_index], + "key_apis": parse_key_apis(row[key_api_index]), + "platform_unit": column_name, + "target_project": PLATFORM_PROJECTS[column_name], + "status": status, + "status_note": status_note, + "severity": severity, + "priority": priority, + "reference_candidates": find_reference_candidates(header, row, column_name), + "notes": row[notes_index] if notes_index is not None else "", + } + ) + + if errors: + raise ValueError("\n".join(errors)) + execution_units.sort(key=lambda unit: (unit["priority"], unit["feature"], unit["platform_unit"])) + return {"execution_units": execution_units} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--matrix", default=str(DEFAULT_MATRIX), help="Path to case-maintenance-matrix.md") + args = parser.parse_args(argv) + + try: + payload = generate_execution_units(Path(args.matrix)) + except (OSError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + print(json.dumps(payload, indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/ai-engineering/tools/generate_case_backlog_test.py b/docs/ai-engineering/tools/generate_case_backlog_test.py new file mode 100644 index 000000000..447190e15 --- /dev/null +++ b/docs/ai-engineering/tools/generate_case_backlog_test.py @@ -0,0 +1,124 @@ +import json +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[3] +GENERATOR = REPO_ROOT / "docs/ai-engineering/tools/generate_case_backlog.py" + + +class CaseBacklogGeneratorTest(unittest.TestCase): + def run_generator(self, matrix_text): + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as handle: + handle.write(matrix_text) + matrix_path = handle.name + try: + return subprocess.run( + [sys.executable, str(GENERATOR), "--matrix", matrix_path], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + finally: + Path(matrix_path).unlink(missing_ok=True) + + def test_generates_execution_units_for_missing_and_partial_cells(self): + matrix = textwrap.dedent( + """ + | Feature | SDK Family | Key APIs | Android full | Android Compose | Windows | Notes | + | --- | --- | --- | --- | --- | --- | --- | + | Join channel audio | Full RTC | `joinChannel`, `setAudioProfile` | `DONE(basic/JoinChannelAudio.java)` | `DONE(samples/JoinChannelAudio.kt)` | `MISSING` | Windows has no basic audio-only join case. | + | Media metadata | Full RTC | `registerMediaMetadataObserver` | `DONE(advanced/MediaMetadata.java)` | `DONE(samples/MediaMetadata.kt)` | `PARTIAL(Advanced/Metadata; smoke pending)` | Runtime metadata smoke pending. | + | Audio mixing | Full RTC | `startAudioMixing` | `DONE(advanced/PlayAudioFiles.java)` | `DONE(samples/PlayAudioFiles.kt)` | `DONE(Advanced/AudioMixing/)` | Full coverage. | + + ## Confirmed Gaps + + | Gap | Affected Units | Severity | + | --- | --- | --- | + | Basic audio-only join channel | Windows | High - missing foundational case | + """ + ) + + result = self.run_generator(matrix) + + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(len(payload["execution_units"]), 2) + self.assertEqual(payload["execution_units"][0]["feature"], "Join channel audio") + self.assertEqual(payload["execution_units"][0]["platform_unit"], "Windows") + self.assertEqual(payload["execution_units"][0]["target_project"], "windows/") + self.assertEqual(payload["execution_units"][0]["status"], "MISSING") + self.assertEqual( + payload["execution_units"][0]["key_apis"], + ["joinChannel", "setAudioProfile"], + ) + self.assertEqual(payload["execution_units"][0]["severity"], "High") + self.assertEqual(payload["execution_units"][0]["priority"], 10) + self.assertEqual( + payload["execution_units"][0]["reference_candidates"], + [ + { + "platform_unit": "Android full", + "project": "Android/APIExample/", + "path": "basic/JoinChannelAudio.java", + }, + { + "platform_unit": "Android Compose", + "project": "Android/APIExample-Compose/", + "path": "samples/JoinChannelAudio.kt", + } + ], + ) + self.assertEqual(payload["execution_units"][1]["platform_unit"], "Windows") + self.assertEqual(payload["execution_units"][1]["status"], "PARTIAL") + self.assertEqual(payload["execution_units"][1]["status_note"], "Advanced/Metadata; smoke pending") + self.assertEqual(payload["execution_units"][1]["severity"], "Unspecified") + self.assertEqual(payload["execution_units"][1]["priority"], 95) + + def test_unknown_platform_column_fails(self): + matrix = textwrap.dedent( + """ + | Feature | SDK Family | Key APIs | Unknown Platform | Notes | + | --- | --- | --- | --- | --- | + | Feature A | Full RTC | `apiA` | `MISSING` | Missing. | + """ + ) + + result = self.run_generator(matrix) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("Unknown Platform", result.stderr) + + def test_confirmed_gap_alias_applies_priority(self): + matrix = textwrap.dedent( + """ + | Feature | SDK Family | Key APIs | Android full | Windows | Notes | + | --- | --- | --- | --- | --- | --- | + | Join channel audio | Full RTC | `joinChannel`, `setAudioProfile` | `DONE(basic/JoinChannelAudio.java)` | `MISSING` | Windows has no basic audio-only join case. | + + ## Confirmed Gaps + + | Gap | Affected Units | Severity | + | --- | --- | --- | + | Basic audio-only join channel | Windows | High - missing foundational case | + """ + ) + + result = self.run_generator(matrix) + + self.assertEqual(result.returncode, 0, result.stderr) + unit = json.loads(result.stdout)["execution_units"][0] + self.assertEqual(unit["feature"], "Join channel audio") + self.assertEqual(unit["platform_unit"], "Windows") + self.assertEqual(unit["severity"], "High") + self.assertEqual(unit["priority"], 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ai-engineering/tools/orchestrate_case_execution.py b/docs/ai-engineering/tools/orchestrate_case_execution.py new file mode 100644 index 000000000..a73b353d9 --- /dev/null +++ b/docs/ai-engineering/tools/orchestrate_case_execution.py @@ -0,0 +1,1586 @@ +#!/usr/bin/env python3 +"""Run one cross-platform API Examples requirement workflow.""" + +import argparse +import concurrent.futures +import hashlib +import json +import os +import signal +import stat +import subprocess +import sys +from pathlib import Path + +from generate_case_backlog import parse_matrix_cell, split_markdown_row +from prepare_case_execution import ( + DEFAULT_REPOSITORY_PROFILE, + PLATFORMS, + collect_sdk_version_checks, + load_repository_profile, + prepare_case_execution, +) +from validate_acceptance_manifest import ( + PLATFORM_ROOTS, + is_platform_build_command, + path_is_within, + validate_contract, + validate_evidence_files, + validate_manifest, + validate_platform_implementation, + validate_platform_verification, +) + + +ROLE_ORDER = ["contract", "implementation", "verification"] +PHASE_ORDER = ["contract", "implementation", "verification"] +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_ROUTING_CONFIG = REPO_ROOT / "docs/ai-engineering/role-routing.json" +COMPLETED_DEPENDENCY_STATUSES = {"PASS", "WAIVED"} +RESULT_VALUES = ["PASS", "FAIL", "BLOCKED", "SKIPPED"] +ROLE_STATUS_VALUES = ["PASS", "FAIL", "BLOCKED", "WAIVED"] + +STRING = {"type": "string"} +STRING_LIST = {"type": "array", "items": STRING} +RESULT = {"type": "string", "enum": RESULT_VALUES} +REFERENCE = { + "type": "object", + "additionalProperties": False, + "required": ["required", "source_case", "contract_result"], + "properties": { + "required": {"type": "boolean"}, + "source_case": STRING, + "contract_result": RESULT, + }, +} +PLATFORM_TARGET = { + "type": "object", + "additionalProperties": False, + "required": [ + "required", + "target_project", + "key_constraints", + "files_allowed", + "waiver_reason", + ], + "properties": { + "required": {"type": "boolean"}, + "target_project": STRING, + "key_constraints": STRING_LIST, + "files_allowed": STRING_LIST, + "waiver_reason": STRING, + }, +} +PLATFORM_TARGETS = { + "type": "object", + "additionalProperties": False, + "required": PLATFORMS, + "properties": {platform: PLATFORM_TARGET for platform in PLATFORMS}, +} +MATRIX_UPDATE = { + "type": "object", + "additionalProperties": False, + "required": ["feature", "platform_unit", "from", "to", "to_cell", "evidence"], + "properties": { + "feature": STRING, + "platform_unit": STRING, + "from": STRING, + "to": STRING, + "to_cell": STRING, + "evidence": STRING, + }, +} +COMMAND_LIST = { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": ["kind", "command", "result"], + "properties": { + "kind": {"type": "string", "enum": ["build", "test", "static", "runtime"]}, + "command": STRING, + "result": RESULT, + }, + }, +} +SKIPPED_CHECK_LIST = { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": ["name", "reason"], + "properties": {"name": STRING, "reason": STRING}, + }, +} +ROLE_OUTPUT_PROPERTIES = { + "contract": { + "scenario": STRING, + "key_apis": STRING_LIST, + "non_goals": STRING_LIST, + "reference": REFERENCE, + "cross_platform_requirements": STRING_LIST, + "platform_targets": PLATFORM_TARGETS, + }, + "implementation": { + "target_project": STRING, + "query_cases": STRING, + "upsert_case": STRING, + "files_changed": STRING_LIST, + "matrix_updates": {"type": "array", "items": MATRIX_UPDATE}, + }, + "verification": { + "result": RESULT, + "findings": STRING_LIST, + "parity_result": RESULT, + "entry_point": STRING, + "ux_notes": STRING, + "commands": COMMAND_LIST, + "build_result": RESULT, + "skipped_checks": SKIPPED_CHECK_LIST, + }, +} + + +def write_json(path, payload): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def read_json(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def load_role_routing(path): + routing = read_json(path) + if routing.get("version") != 1: + raise ValueError("role routing version must be 1") + profiles = routing.get("profiles") + roles = routing.get("roles") + phases = routing.get("phases") + if not isinstance(profiles, dict) or not profiles: + raise ValueError("role routing profiles must be a non-empty object") + if not isinstance(roles, dict) or set(roles) != set(ROLE_ORDER): + raise ValueError(f"role routing must define exactly: {', '.join(ROLE_ORDER)}") + if phases != PHASE_ORDER: + raise ValueError(f"role routing phases must be: {', '.join(PHASE_ORDER)}") + for role, route in roles.items(): + if route.get("phase") != role: + raise ValueError(f"role routing {role}.phase must be {role}") + profile = route.get("profile") + if profile not in profiles: + raise ValueError(f"role routing {role}.profile={profile} is not defined") + if "reasoning_effort" not in profiles[profile]: + raise ValueError(f"role routing profile {profile} needs reasoning_effort") + return routing + + +def init_workspace(args): + run_dir = Path(args.run_dir).resolve() + run_dir.mkdir(parents=True, exist_ok=True) + matrix_path = Path(args.matrix).resolve() + routing_path = Path(args.routing_config).resolve() + profile_path = Path(args.repository_profile).resolve() + routing = load_role_routing(routing_path) + load_repository_profile(profile_path) + package = prepare_case_execution( + matrix_path, + args.feature, + index=args.index, + sdk_family=args.sdk_family, + key_apis=args.key_api, + target_sdk_version=args.target_sdk_version, + repository_profile=profile_path, + ) + package["matrix_path"] = stable_matrix_path(matrix_path) + package["routing_config"] = repo_relative_path(routing_path) + package["routing_config_sha256"] = sha256_file(routing_path) + package["repository_profile"] = repo_relative_path(profile_path) + package["repository_profile_sha256"] = sha256_file(profile_path) + write_json(run_dir / "execution-package.json", package) + write_json(run_dir / "acceptance-manifest.json", package["acceptance_manifest_seed"]) + + contract = package["acceptance_manifest_seed"]["contract"] + write_artifact_seed(run_dir, "contract", "contract", None, contract) + write_role_schema(run_dir / "role-schemas/contract.json", "contract") + write_prompt(run_dir, "contract", "contract", None, package, routing, {}) + for platform in PLATFORMS: + unit = package["acceptance_manifest_seed"]["platforms"][platform] + for role in ["implementation", "verification"]: + name = artifact_name(role, platform) + write_artifact_seed(run_dir, name, role, platform, unit[role]) + write_role_schema(run_dir / "role-schemas" / f"{name}.json", role) + write_prompt(run_dir, name, role, platform, package, routing, {}) + print(f"Execution workspace initialized: {run_dir}") + return 0 + + +def write_artifact_seed(run_dir, name, role, platform, artifact): + payload = {"name": name, "role": role, **artifact} + if platform: + payload["platform"] = platform + write_json(run_dir / "role-artifacts" / f"{name}.json", payload) + + +def write_role_schema(path, role): + output_properties = ROLE_OUTPUT_PROPERTIES[role] + schema = { + "type": "object", + "additionalProperties": False, + "required": ["status", "evidence", "summary", "output"], + "properties": { + "status": {"type": "string", "enum": ROLE_STATUS_VALUES}, + "evidence": STRING, + "summary": STRING, + "waiver_reason": STRING, + "output": { + "type": "object", + "additionalProperties": False, + "required": list(output_properties), + "properties": output_properties, + }, + }, + } + write_json(path, schema) + + +def write_prompt(run_dir, name, role, platform, package, routing, dependencies): + requirement = package["requirement"] + target = "all official platforms" + if platform: + target = resolved_platform_target(package, platform, dependencies) + host_note = host_constraint(platform) if role == "verification" else "" + waiver_note = "" + if platform: + waiver_note = ( + "If Contract marks this platform required=false, do not modify or verify it. " + "Return status=WAIVED, copy Contract's waiver_reason into the top-level " + "waiver_reason field, use SKIPPED for Verification result/parity_result/build_result, " + "and report no executed build command." + ) + body = f"""# {name} Agent Task + +Requirement: {requirement['feature']} +SDK family: {requirement['sdk_family']} +Key APIs: {', '.join(requirement['key_apis'])} +Target SDK version: {requirement['target_sdk_version']} +Role: {role} +Platform: {platform or 'shared'} +Target: {target} +Model profile: {routing['roles'][role]['profile']} + +Required output: +{json.dumps(package['role_contracts'][role], indent=2, ensure_ascii=False)} + +Dependency artifacts: +{format_dependency_context(dependencies)} + +The dispatcher starts platform work in the Contract-selected target so nested AGENTS.md files load automatically. Confirm those instructions before acting. Keep platform source changes inside the Contract target and never share source files across platform roots. + +{host_note} + +For Verification, report every executed check with kind=build|test|static|runtime and copy the command string exactly from Codex command telemetry. The dispatcher binds the declared result to the JSONL exit code. + +{waiver_note} + +Return only the JSON object required by the supplied output schema. The dispatcher records identity and provenance. +""" + path = run_dir / "role-prompts" / f"{name}.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + return path + + +def host_constraint(platform): + if platform == "windows" and sys.platform != "win32": + host = "macOS" if sys.platform == "darwin" else sys.platform + return ( + f"This host is {host}. {host} cannot provide Windows build evidence. " + "Do not download Windows SDK archives, emulate Windows, cross-compile, or use a " + "substitute compiler as platform evidence. Run repository-local static checks only " + "and report unavailable Windows build/runtime evidence as BLOCKED." + ) + return "" + + +def resolved_platform_target(package, platform, dependencies): + contract = dependencies.get("contract", {}) + output = contract.get("output", {}) if isinstance(contract, dict) else {} + targets = output.get("platform_targets", {}) if isinstance(output, dict) else {} + if platform in targets and targets[platform].get("target_project"): + return targets[platform]["target_project"] + return package["acceptance_manifest_seed"]["contract"]["output"]["platform_targets"][platform][ + "target_project" + ] + + +def format_dependency_context(dependencies): + if not dependencies: + return "None." + sections = [] + for name, artifact in dependencies.items(): + payload = { + "status": artifact.get("status"), + "summary": artifact.get("summary"), + "output": artifact.get("output"), + } + sections.append(f"- {name}:\n```json\n{json.dumps(payload, indent=2, ensure_ascii=False)}\n```") + return "\n".join(sections) + + +def dispatch_workspace(args): + run_dir = Path(args.run_dir).resolve() + package = read_json(run_dir / "execution-package.json") + routing_path = Path(args.routing_config).resolve() + routing = load_role_routing(routing_path) + validate_execution_configuration(package, routing_path) + specs = dispatch_specs(run_dir, args.phase, args.platform, args.retry) + route = routing["roles"][args.phase] + profile = route["profile"] + model = resolve_model(profile, args.model) + codex_version = "dry-run" if args.dry_run else read_codex_version(args.codex_bin) + tasks = [] + for spec in specs: + dependencies = dependencies_for_spec(run_dir, spec) + working_directory = resolve_working_directory(package, spec, dependencies) + resolved = { + "version": 1, + "routing_config": repo_relative_path(routing_path), + "routing_config_sha256": sha256_file(routing_path), + "phase": args.phase, + "name": spec["name"], + "platform": spec["platform"], + "profile": profile, + "model": model, + "reasoning_effort": routing["profiles"][profile]["reasoning_effort"], + "codex_version": codex_version, + "working_directory": repo_relative_path(working_directory), + } + write_json(run_dir / "resolved-role-routing" / f"{spec['name']}.json", resolved) + snapshot_path, snapshot_sha256 = write_input_snapshot( + run_dir, args.phase, spec["name"], dependencies, resolved + ) + input_snapshot = read_json(snapshot_path) + expected_repository_state = input_snapshot["repository_state"] + prompt_path = write_prompt( + run_dir, spec["name"], spec["role"], spec["platform"], package, routing, dependencies + ) + schema_path = run_dir / "role-schemas" / f"{spec['name']}.json" + write_role_schema(schema_path, spec["role"]) + raw_result_path = run_dir / "raw-results" / f"{spec['name']}.json" + command = build_codex_command( + args.codex_bin, + route, + model, + routing["profiles"][profile]["reasoning_effort"], + schema_path, + raw_result_path, + working_directory, + ) + tasks.append( + { + **spec, + "command": command, + "prompt_path": prompt_path, + "artifact_path": run_dir / "role-artifacts" / f"{spec['name']}.json", + "raw_result_path": raw_result_path, + "log_path": run_dir / "dispatch-logs" / f"{spec['name']}.jsonl", + "stderr_path": run_dir / "dispatch-logs" / f"{spec['name']}.stderr.log", + "model_profile": profile, + "model": model, + "reasoning_effort": routing["profiles"][profile]["reasoning_effort"], + "codex_version": codex_version, + "working_directory": working_directory, + "working_directory_manifest": repo_relative_path(working_directory), + "input_snapshot": run_relative_path(snapshot_path, run_dir), + "input_snapshot_sha256": snapshot_sha256, + "expected_repository_state": expected_repository_state, + "snapshot_dependencies": dependencies, + "resolved_routing": resolved, + "previous_files_changed": ( + read_json(run_dir / "role-artifacts" / f"{spec['name']}.json") + .get("output", {}) + .get("files_changed", []) + if spec["role"] == "implementation" + else [] + ), + "timeout_seconds": args.timeout_seconds, + } + ) + if args.dry_run: + print( + json.dumps( + { + "phase": args.phase, + "commands": [ + { + "name": task["name"], + "platform": task["platform"], + "command": task["command"], + } + for task in tasks + ], + }, + indent=2, + ensure_ascii=False, + ) + ) + return 0 + if args.phase == "implementation": + for task in tasks: + invalidate_platform_verification(run_dir, package, task["platform"]) + run_implementation_tasks(run_dir, tasks) + elif args.phase == "verification": + run_verification_tasks(run_dir, tasks) + else: + run_tasks(tasks) + if args.phase == "contract": + validate_completed_contract(run_dir) + print(f"Codex phase completed: {args.phase} ({', '.join(task['name'] for task in tasks)})") + return 0 + + +def dispatch_specs(run_dir, phase, requested_platforms, retry): + if phase == "contract": + if requested_platforms: + raise ValueError("--platform is not valid for the shared contract phase") + candidates = [{"name": "contract", "role": "contract", "platform": None}] + else: + platforms = requested_platforms or PLATFORMS + candidates = [ + {"name": artifact_name(phase, platform), "role": phase, "platform": platform} + for platform in platforms + ] + selected = [] + for spec in candidates: + path = run_dir / "role-artifacts" / f"{spec['name']}.json" + if not path.exists(): + raise ValueError(f"missing role artifact: {spec['name']}") + artifact = read_json(path) + already_dispatched = artifact.get("dispatch", {}).get("mode") == "codex-exec" + status = artifact.get("status") + if already_dispatched: + if retry and status in {"FAIL", "BLOCKED"}: + selected.append(spec) + elif requested_platforms or phase == "contract": + raise ValueError(f"role {spec['name']} has already been dispatched with status {status}") + continue + selected.append(spec) + if not selected: + raise ValueError(f"all {phase} role artifacts are already complete") + return selected + + +def validate_completed_contract(run_dir): + artifact_path = run_dir / "role-artifacts/contract.json" + artifact = read_json(artifact_path) + if artifact.get("status") != "PASS": + if artifact.get("status") not in {"FAIL", "BLOCKED"}: + artifact["status"] = "BLOCKED" + artifact["evidence"] = "Contract is required and cannot be waived." + artifact["summary"] = "Contract requires correction before platform implementation." + write_json(artifact_path, artifact) + raise ValueError(f"Contract gate returned status {artifact.get('status')}") + manifest = read_json(run_dir / "acceptance-manifest.json") + manifest["contract"] = strip_artifact_metadata(artifact) + errors = [] + validate_contract(manifest, errors, {}) + if not errors: + return + artifact["status"] = "BLOCKED" + artifact["evidence"] = "Contract semantic gate failed: " + "; ".join(errors) + artifact["summary"] = "Contract requires correction before platform implementation." + write_json(artifact_path, artifact) + raise ValueError("Contract semantic gate failed: " + "; ".join(errors)) + + +def dependencies_for_spec(run_dir, spec): + dependency_names = [] + if spec["role"] == "implementation": + dependency_names = ["contract"] + elif spec["role"] == "verification": + dependency_names = ["contract", artifact_name("implementation", spec["platform"])] + dependencies = {} + for name in dependency_names: + if name == "contract": + validate_completed_contract(run_dir) + if name.endswith("-implementation"): + validate_completed_implementation(run_dir, spec["platform"]) + path = run_dir / "role-artifacts" / f"{name}.json" + if not path.exists(): + raise ValueError(f"role {spec['name']} requires missing dependency artifact: {name}") + artifact = read_json(path) + status = artifact.get("status") + allowed_statuses = {"PASS"} if name == "contract" else COMPLETED_DEPENDENCY_STATUSES + if status not in allowed_statuses: + raise ValueError( + f"role {spec['name']} requires {name} status " + f"{'PASS' if name == 'contract' else 'PASS or WAIVED'}, got {status}" + ) + dependencies[name] = artifact + return dependencies + + +def invalidate_platform_verification(run_dir, package, platform): + seed = package["acceptance_manifest_seed"]["platforms"][platform]["verification"] + name = artifact_name("verification", platform) + write_artifact_seed(run_dir, name, "verification", platform, seed) + + +def resolve_working_directory(package, spec, dependencies): + if not spec["platform"]: + return REPO_ROOT + target = resolved_platform_target(package, spec["platform"], dependencies) + platform_root = PLATFORM_ROOTS[spec["platform"]] + if not path_is_within(target, platform_root): + raise ValueError( + f"{spec['platform']} target must be inside {platform_root}: {target}" + ) + path = (REPO_ROOT / target).resolve() + try: + path.relative_to(REPO_ROOT) + except ValueError as exc: + raise ValueError(f"platform target must be inside the repository: {target}") from exc + if not path.is_dir(): + raise ValueError(f"platform target directory does not exist: {target}") + return path + + +def run_tasks(tasks): + errors = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(tasks), len(PLATFORMS))) as executor: + future_tasks = {executor.submit(run_codex_role, **task_args(task)): task for task in tasks} + for future in concurrent.futures.as_completed(future_tasks): + task = future_tasks[future] + try: + future.result() + except (OSError, ValueError, json.JSONDecodeError) as exc: + errors.append((task["name"], str(exc))) + if errors: + errors.sort() + raise ValueError("; ".join(detail for _, detail in errors)) + + +def run_verification_tasks(run_dir, tasks): + errors = [] + try: + run_tasks(tasks) + except ValueError as exc: + errors.append(str(exc)) + for task in tasks: + try: + validate_completed_verification(run_dir, task["platform"]) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + errors.append(str(exc)) + if errors: + raise ValueError("; ".join(errors)) + + +def run_implementation_tasks(run_dir, tasks): + errors = [] + for task in tasks: + refresh_implementation_input_snapshot(run_dir, task) + before = collect_repository_fingerprints() + try: + run_codex_role(**task_args(task)) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + errors.append((task["name"], str(exc))) + try: + reconcile_implementation_changes(run_dir, [task], before) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + errors.append((task["name"], str(exc))) + try: + validate_completed_implementation(run_dir, task["platform"]) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + errors.append((task["name"], str(exc))) + if errors: + errors.sort() + raise ValueError("; ".join(detail for _, detail in errors)) + + +def validate_completed_implementation(run_dir, platform): + artifact_path = run_dir / "role-artifacts" / f"{platform}-implementation.json" + artifact = read_json(artifact_path) + contract = read_json(run_dir / "role-artifacts/contract.json") + target = contract.get("output", {}).get("platform_targets", {}).get(platform, {}) + required = bool(target.get("required", True)) + expected_status = "PASS" if required else "WAIVED" + errors = [] + if artifact.get("status") != expected_status: + errors.append( + f"{platform} Implementation gate requires status {expected_status}, " + f"got {artifact.get('status')}" + ) + manifest = read_json(run_dir / "acceptance-manifest.json") + manifest["final_status"] = "PASS" + verification_stub = { + "build_result": "PASS", + "parity_result": "PASS", + "skipped_checks": [], + } + output = artifact.get("output") + if not isinstance(output, dict) or not output: + errors.append(f"{platform} Implementation output must be a non-empty object") + else: + validate_platform_implementation( + platform, + target, + output, + verification_stub, + manifest, + errors, + ) + if not errors: + return + if artifact.get("status") not in {"FAIL", "BLOCKED"}: + artifact["status"] = "BLOCKED" + artifact["evidence"] = "Implementation semantic gate failed: " + "; ".join(errors) + artifact["summary"] = "Implementation requires correction before Verification." + write_json(artifact_path, artifact) + raise ValueError("Implementation semantic gate failed: " + "; ".join(errors)) + + +def validate_completed_verification(run_dir, platform): + artifact_path = run_dir / "role-artifacts" / f"{platform}-verification.json" + artifact = read_json(artifact_path) + contract = read_json(run_dir / "role-artifacts/contract.json") + target = contract.get("output", {}).get("platform_targets", {}).get(platform, {}) + required = bool(target.get("required", True)) + status = artifact.get("status") + if required and status in {"FAIL", "BLOCKED"}: + return + errors = [] + if required and status != "PASS": + errors.append(f"{platform} Verification gate requires PASS, FAIL, or BLOCKED, got {status}") + if not required and status != "WAIVED": + errors.append(f"{platform} Verification gate requires status WAIVED, got {status}") + manifest = read_json(run_dir / "acceptance-manifest.json") + manifest["final_status"] = "PASS WITH RISKS" + output = artifact.get("output") + if not isinstance(output, dict) or not output: + errors.append(f"{platform} Verification output must be a non-empty object") + else: + validate_platform_verification( + platform, + required, + output, + artifact.get("dispatch", {}), + manifest, + errors, + ) + if not errors: + return + if status not in {"FAIL", "BLOCKED"}: + artifact["status"] = "BLOCKED" + artifact["evidence"] = "Verification semantic gate failed: " + "; ".join(errors) + artifact["summary"] = "Verification requires correction before acceptance." + write_json(artifact_path, artifact) + raise ValueError("Verification semantic gate failed: " + "; ".join(errors)) + + +def refresh_implementation_input_snapshot(run_dir, task): + snapshot_path, snapshot_sha256 = write_input_snapshot( + run_dir, + "implementation", + task["name"], + task["snapshot_dependencies"], + task["resolved_routing"], + ) + task["input_snapshot"] = run_relative_path(snapshot_path, run_dir) + task["input_snapshot_sha256"] = snapshot_sha256 + task["expected_repository_state"] = read_json(snapshot_path)["repository_state"] + + +def task_args(task): + allowed = { + "command", + "name", + "role", + "platform", + "prompt_path", + "artifact_path", + "raw_result_path", + "log_path", + "stderr_path", + "model_profile", + "model", + "reasoning_effort", + "codex_version", + "working_directory", + "working_directory_manifest", + "input_snapshot", + "input_snapshot_sha256", + "expected_repository_state", + "timeout_seconds", + } + return {key: value for key, value in task.items() if key in allowed} + + +def resolve_bound_repository_profile(package): + profile_value = package.get("repository_profile") + if not isinstance(profile_value, str) or not profile_value: + raise ValueError("execution package repository profile is missing; re-run init") + profile_path = (REPO_ROOT / profile_value).resolve() + try: + profile_path.relative_to(REPO_ROOT.resolve()) + except ValueError as exc: + raise ValueError("execution package repository profile must be inside the repository") from exc + if package.get("repository_profile_sha256") != sha256_file(profile_path): + raise ValueError("repository profile content changed since init; re-run init") + load_repository_profile(profile_path) + return profile_path + + +def validate_execution_configuration(package, routing_path): + if package.get("routing_config") != repo_relative_path(routing_path): + raise ValueError("routing config path changed since init; re-run init") + if package.get("routing_config_sha256") != sha256_file(routing_path): + raise ValueError("routing config content changed since init; re-run init") + resolve_bound_repository_profile(package) + + +def resolve_model(profile, override): + model = override or os.environ.get(f"CODEX_MODEL_{profile.upper()}") + if not model: + raise ValueError(f"model is required; pass --model or set CODEX_MODEL_{profile.upper()}") + return model + + +def read_codex_version(codex_bin): + try: + result = subprocess.run( + [codex_bin, "--version"], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=10, + ) + except subprocess.TimeoutExpired as exc: + raise ValueError("timed out while reading Codex version") from exc + if result.returncode != 0: + raise ValueError(f"failed to read Codex version: {result.stderr.strip()}") + if not result.stdout.strip(): + raise ValueError("Codex version output is empty") + return result.stdout.strip() + + +def write_input_snapshot(run_dir, phase, name, dependencies, resolved_routing): + package = read_json(run_dir / "execution-package.json") + dependency_hashes = { + dependency: sha256_file(run_dir / "role-artifacts" / f"{dependency}.json") + for dependency in dependencies + } + snapshot = { + "version": 1, + "phase": phase, + "name": name, + "execution_package_sha256": sha256_file(run_dir / "execution-package.json"), + "repository_state": repository_state(), + "dependency_artifact_sha256": dependency_hashes, + "routing_config": resolved_routing["routing_config"], + "routing_config_sha256": resolved_routing["routing_config_sha256"], + "repository_profile": package["repository_profile"], + "repository_profile_sha256": package["repository_profile_sha256"], + } + serialized = json.dumps(snapshot, indent=2, ensure_ascii=False) + "\n" + snapshot_sha256 = sha256_text(serialized) + path = run_dir / "input-snapshots" / f"{name}-{snapshot_sha256[:12]}.json" + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_text(encoding="utf-8") != serialized: + raise ValueError(f"input snapshot hash collision: {path}") + path.write_text(serialized, encoding="utf-8") + return path, snapshot_sha256 + + +def run_git(arguments): + result = subprocess.run( + ["git", *arguments], cwd=REPO_ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if result.returncode != 0: + raise ValueError(f"git {' '.join(arguments)} failed: {result.stderr.strip()}") + return result.stdout.rstrip("\n") + + +def run_git_bytes(arguments): + result = subprocess.run( + ["git", *arguments], cwd=REPO_ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if result.returncode != 0: + detail = result.stderr.decode("utf-8", errors="replace").strip() + raise ValueError(f"git {' '.join(arguments)} failed: {detail}") + return result.stdout + + +def collect_repository_fingerprints(repo_root=REPO_ROOT, include_tracked=True): + repo_root = Path(repo_root) + fingerprints = {} + if include_tracked: + staged = run_git_bytes_at(repo_root, ["ls-files", "--stage", "-z"]) + for record in filter(None, staged.split(b"\0")): + metadata, separator, raw_path = record.partition(b"\t") + if not separator: + continue + mode, object_id, stage = metadata.split() + path_text = os.fsdecode(raw_path) + fingerprints[path_text] = { + "type": "index", + "mode": mode.decode("ascii"), + "object_id": object_id.decode("ascii"), + "stage": stage.decode("ascii"), + } + dirty = run_git_bytes_at(repo_root, ["diff", "--name-only", "-z"]) + for raw_path in filter(None, dirty.split(b"\0")): + path_text = os.fsdecode(raw_path) + fingerprints[path_text] = fingerprint_repository_path(repo_root / path_text) + + untracked = run_git_bytes_at( + repo_root, ["ls-files", "-z", "--others", "--exclude-standard"] + ) + for raw_path in filter(None, untracked.split(b"\0")): + path_text = os.fsdecode(raw_path) + fingerprints[path_text] = fingerprint_repository_path(repo_root / path_text) + return dict(sorted(fingerprints.items())) + + +def run_git_bytes_at(repo_root, arguments): + result = subprocess.run( + ["git", *arguments], cwd=repo_root, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if result.returncode != 0: + detail = result.stderr.decode("utf-8", errors="replace").strip() + raise ValueError(f"git {' '.join(arguments)} failed: {detail}") + return result.stdout + + +def fingerprint_repository_path(path): + try: + metadata = path.lstat() + except FileNotFoundError: + return {"type": "missing", "mode": "", "sha256": ""} + mode = oct(stat.S_IMODE(metadata.st_mode)) + if path.is_symlink(): + return { + "type": "symlink", + "mode": mode, + "sha256": sha256_text(os.readlink(path)), + } + if path.is_file(): + return {"type": "file", "mode": mode, "sha256": sha256_file(path)} + return {"type": "other", "mode": mode, "sha256": ""} + + +def repository_state(): + return { + "head": run_git(["rev-parse", "HEAD"]), + "status": run_git(["status", "--short"]), + "diff_sha256": sha256_bytes(run_git_bytes(["diff", "--binary", "HEAD"])), + "untracked_files": collect_repository_fingerprints(include_tracked=False), + } + + +def changed_repository_paths(before, after): + return sorted( + path + for path in set(before) | set(after) + if before.get(path) != after.get(path) + ) + + +def normalize_repo_path_text(path_text): + path = Path(path_text) + if path.is_absolute(): + return repo_relative_path(path) + normalized = str(path_text).replace("\\", "/") + return normalized[2:] if normalized.startswith("./") else normalized + + +def reconcile_implementation_changes(run_dir, tasks, before): + after = collect_repository_fingerprints() + changed = changed_repository_paths(before, after) + dispatched_platforms = {task["platform"] for task in tasks} + unexpected = [ + path + for path in changed + if not any( + platform in dispatched_platforms and path_is_within(path, root) + for platform, root in PLATFORM_ROOTS.items() + ) + ] + contract = read_json(run_dir / "role-artifacts/contract.json") + targets = contract.get("output", {}).get("platform_targets", {}) + scope_errors = [] + for task in tasks: + platform = task["platform"] + platform_changes = [ + path for path in changed if path_is_within(path, PLATFORM_ROOTS[platform]) + ] + allowed = targets.get(platform, {}).get("files_allowed", []) + outside_allowed = [ + path + for path in platform_changes + if not any(path_is_within(path, allowed_path) for allowed_path in allowed) + ] + if outside_allowed: + scope_errors.append( + f"{platform} Implementation modified files outside Contract files_allowed: " + + ", ".join(outside_allowed) + ) + + artifact_path = run_dir / "role-artifacts" / f"{task['name']}.json" + artifact = read_json(artifact_path) + previous = { + normalize_repo_path_text(path) for path in task.get("previous_files_changed", []) + } + delta_path = run_dir / "repository-deltas" / f"{task['name']}.json" + prior_delta = read_json(delta_path) if delta_path.exists() else {} + before_values = dict(prior_delta.get("before", {})) + after_values = dict(prior_delta.get("after", {})) + for path in previous: + before_values.setdefault(path, None) + after_values.setdefault(path, after.get(path)) + for path in platform_changes: + before_values.setdefault(path, before.get(path)) + after_values[path] = after.get(path) + cumulative_paths = previous | set(platform_changes) | set(prior_delta.get("changed_files", [])) + net_changes = sorted( + path for path in cumulative_paths if before_values.get(path) != after_values.get(path) + ) + artifact["output"]["files_changed"] = net_changes + current_dispatch = artifact.get("dispatch", {}) + current_snapshot = current_dispatch.get("input_snapshot_sha256") + attempt = { + "input_snapshot": task.get("input_snapshot"), + "input_snapshot_sha256": task.get("input_snapshot_sha256"), + "run_id": ( + current_dispatch.get("run_id") + if current_snapshot == task.get("input_snapshot_sha256") + else "" + ), + "status": ( + artifact.get("status") + if current_snapshot == task.get("input_snapshot_sha256") + else "DISPATCH_ERROR" + ), + "changed_files": platform_changes, + } + delta = { + "version": 2, + "platform": platform, + "changed_files": net_changes, + "before": {path: before_values.get(path) for path in net_changes}, + "after": {path: after_values.get(path) for path in net_changes}, + "attempts": [*prior_delta.get("attempts", []), attempt], + } + write_json(delta_path, delta) + artifact["dispatch"]["repository_delta"] = run_relative_path(delta_path, run_dir) + artifact["dispatch"]["repository_delta_sha256"] = sha256_file(delta_path) + write_json(artifact_path, artifact) + if unexpected: + violation_path = run_dir / "repository-deltas/implementation-scope-violations.json" + write_json( + violation_path, + { + "version": 1, + "changed_files": unexpected, + "before": {path: before.get(path) for path in unexpected}, + "after": {path: after.get(path) for path in unexpected}, + }, + ) + scope_errors.append( + "Implementation modified files outside dispatched platform roots: " + + ", ".join(unexpected) + ) + if scope_errors: + raise ValueError("; ".join(scope_errors)) + + +def build_codex_command( + codex_bin, route, model, reasoning_effort, schema_path, raw_result_path, working_directory +): + raw_result_path.parent.mkdir(parents=True, exist_ok=True) + command = [ + codex_bin, + "exec", + "-C", + str(working_directory), + "-s", + route["sandbox"], + ] + command.extend( + [ + "--json", + "--output-schema", + str(schema_path), + "--output-last-message", + str(raw_result_path), + "--model", + model, + "-c", + f'model_reasoning_effort="{reasoning_effort}"', + "-", + ] + ) + return command + + +def run_codex_role( + command, + name, + role, + platform, + prompt_path, + artifact_path, + raw_result_path, + log_path, + stderr_path, + model_profile, + model, + reasoning_effort, + codex_version, + working_directory, + working_directory_manifest, + input_snapshot, + input_snapshot_sha256, + expected_repository_state, + timeout_seconds, +): + prompt = prompt_path.read_text(encoding="utf-8") + log_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Dispatching {name} (timeout {timeout_seconds}s); logs: {log_path}", flush=True) + execution_error = None + with log_path.open("w", encoding="utf-8") as stdout_handle, stderr_path.open( + "w", encoding="utf-8" + ) as stderr_handle: + process = subprocess.Popen( + command, + cwd=working_directory, + stdin=subprocess.PIPE, + stdout=stdout_handle, + stderr=stderr_handle, + text=True, + start_new_session=sys.platform != "win32", + creationflags=( + getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + if sys.platform == "win32" + else 0 + ), + ) + try: + process.communicate(input=prompt, timeout=timeout_seconds) + except subprocess.TimeoutExpired: + terminate_process_tree(process) + process.communicate() + execution_error = ValueError( + f"Codex dispatch for {name} timed out after {timeout_seconds} seconds" + ) + stdout = log_path.read_text(encoding="utf-8") + stderr = stderr_path.read_text(encoding="utf-8") + if execution_error is None and process.returncode != 0: + execution_error = ValueError( + f"Codex dispatch failed for {name}: {(stderr or stdout).strip()}" + ) + mutation_error = None + if role in {"contract", "verification"}: + actual_repository_state = repository_state() + if actual_repository_state != expected_repository_state: + violation_path = ( + artifact_path.parent.parent + / "repository-deltas" + / f"{name}-scope-violation.json" + ) + write_json( + violation_path, + { + "version": 1, + "role": role, + "name": name, + "before": expected_repository_state, + "after": actual_repository_state, + }, + ) + mutation_error = ValueError( + f"Codex {role} run {name} modified repository files" + ) + if execution_error or mutation_error: + details = [str(error) for error in [execution_error, mutation_error] if error] + raise ValueError("; ".join(details)) + run_id = extract_run_id(stdout) + if not run_id: + raise ValueError(f"Codex dispatch for {name} did not report a thread or run id") + agent_result = read_json(raw_result_path) + for field in ["status", "evidence", "summary", "output"]: + if field not in agent_result: + raise ValueError(f"Codex result for {name} is missing {field}") + if role == "verification": + validate_verification_host(platform, agent_result, sys.platform) + bind_verification_command_evidence( + agent_result["output"], + stdout, + run_relative_path(log_path, artifact_path.parent.parent), + platform, + working_directory_manifest, + ) + artifact = { + "name": name, + "role": role, + **({"platform": platform} if platform else {}), + "agent_id": f"{name}-{run_id}", + "dispatch": { + "mode": "codex-exec", + "run_id": run_id, + "prompt": f"role-prompts/{name}.md", + "prompt_sha256": sha256_text(prompt), + "artifact": f"role-artifacts/{name}.json", + "evidence": f"{codex_version} completed {name}; log: dispatch-logs/{name}.jsonl", + "command_log": run_relative_path(log_path, artifact_path.parent.parent), + "command_log_sha256": sha256_file(log_path), + "model_profile": model_profile, + "model": model, + "reasoning_effort": reasoning_effort, + "codex_version": codex_version, + "working_directory": working_directory_manifest, + "host_platform": sys.platform, + "input_snapshot": input_snapshot, + "input_snapshot_sha256": input_snapshot_sha256, + }, + "status": agent_result["status"], + "evidence": agent_result["evidence"], + "summary": agent_result["summary"], + "output": agent_result["output"], + } + if "waiver_reason" in agent_result: + artifact["waiver_reason"] = agent_result["waiver_reason"] + write_json(artifact_path, artifact) + + +def terminate_process_tree(process): + if process.poll() is not None: + return + if sys.platform == "win32": + result = subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(process.pid)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode != 0 and process.poll() is None: + process.kill() + return + try: + os.killpg(os.getpgid(process.pid), signal.SIGKILL) + except ProcessLookupError: + return + + +def validate_verification_host(platform, agent_result, host_platform): + if platform != "windows" or host_platform == "win32": + return + output = agent_result.get("output", {}) + has_build_execution = any( + isinstance(command, dict) + and command.get("kind") == "build" + and command.get("result") in {"PASS", "FAIL"} + for command in output.get("commands", []) + ) + if agent_result.get("status") == "WAIVED": + if output.get("build_result") == "PASS" or has_build_execution: + raise ValueError("WAIVED Windows Verification cannot report build execution") + return + if ( + agent_result.get("status") != "BLOCKED" + or output.get("result") != "BLOCKED" + or output.get("build_result") != "BLOCKED" + or has_build_execution + ): + raise ValueError( + "Windows Verification build evidence requires a Windows host; " + f"current host_platform={host_platform}" + ) + + +def bind_verification_command_evidence( + output, jsonl, log_path, platform, working_directory_manifest +): + commands = output.get("commands", []) if isinstance(output, dict) else [] + events = extract_command_events(jsonl) + used = set() + for index, declared in enumerate(commands): + result = declared.get("result") + if result not in {"PASS", "FAIL"}: + continue + if declared.get("kind") == "build" and not is_platform_build_command( + platform, declared.get("command"), working_directory_manifest + ): + raise ValueError( + f"Verification command {index} is not a recognized {platform} build command: " + f"{declared.get('command')}" + ) + declared_text = normalize_command(declared.get("command")) + match_index = next( + ( + event_index + for event_index, event in enumerate(events) + if event_index not in used and normalize_command(event["command"]) == declared_text + ), + None, + ) + if match_index is None: + raise ValueError( + f"Verification command {index} has no matching command_execution event: {declared.get('command')}" + ) + used.add(match_index) + event = events[match_index] + actual_result = "PASS" if event["exit_code"] == 0 else "FAIL" + if result != actual_result: + raise ValueError( + f"Verification command {declared.get('command')} declared {result} but exit_code={event['exit_code']}" + ) + declared["evidence"] = ( + f"{log_path}#{event['id']}; exit_code={event['exit_code']}" + ) + + +def extract_command_events(jsonl): + events = [] + for line in jsonl.splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + item = event.get("item") + if event.get("type") != "item.completed" or not isinstance(item, dict): + continue + if item.get("type") != "command_execution" or not isinstance(item.get("exit_code"), int): + continue + events.append( + { + "id": str(item.get("id") or f"command-{len(events)}"), + "command": item.get("command"), + "exit_code": item["exit_code"], + } + ) + return events + + +def normalize_command(command): + if isinstance(command, list): + command = " ".join(str(part) for part in command) + return " ".join(str(command or "").split()) + + +def extract_run_id(jsonl): + for line in jsonl.splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + for field in ["thread_id", "run_id", "session_id"]: + if event.get(field): + return str(event[field]) + thread = event.get("thread") + if isinstance(thread, dict) and thread.get("id"): + return str(thread["id"]) + return "" + + +def assemble_workspace(args): + run_dir = Path(args.run_dir).resolve() + scope_violations = sorted( + (run_dir / "repository-deltas").glob("*scope-violation*.json") + ) + if scope_violations: + raise ValueError( + "repository scope violation is unresolved; restore the repository and re-run init" + ) + package = read_json(run_dir / "execution-package.json") + profile_path = resolve_bound_repository_profile(package) + matrix_path = Path(args.matrix).resolve() + if package.get("matrix_path") != stable_matrix_path(matrix_path): + raise ValueError("matrix path changed since init; re-run init") + manifest = read_json(run_dir / "acceptance-manifest.json") + if manifest.get("requirement") != package.get("requirement"): + raise ValueError("requirement changed since init; re-run init") + artifacts = load_role_artifacts(run_dir / "role-artifacts") + manifest["final_status"] = args.final_status + manifest["contract"] = strip_artifact_metadata(artifacts["contract"]) + for platform in PLATFORMS: + manifest["platforms"][platform] = { + role: strip_artifact_metadata(artifacts[artifact_name(role, platform)]) + for role in ["implementation", "verification"] + } + manifest["cross_platform_acceptance"] = { + "result": args.cross_platform_result, + "evidence": args.cross_platform_evidence, + "differences": args.cross_platform_difference, + } + manifest["release"]["checks"] = collect_sdk_version_checks( + manifest["requirement"]["target_sdk_version"], profile_path=profile_path + ) + errors = validate_manifest(manifest) + errors.extend(validate_evidence_files(manifest, run_dir)) + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + updates = [] + for platform in PLATFORMS: + updates.extend(manifest["platforms"][platform]["implementation"]["output"]["matrix_updates"]) + if updates and manifest["final_status"] != "BLOCKED": + update_matrix(matrix_path, updates) + output_path = run_dir / "final-acceptance-manifest.json" + write_json(output_path, manifest) + print(f"Acceptance manifest valid: {output_path}") + return 0 + +def load_role_artifacts(artifact_dir): + artifacts = {} + for artifact_path in sorted(artifact_dir.glob("*.json")): + artifact = read_json(artifact_path) + if not isinstance(artifact, dict): + raise ValueError(f"role artifact must be an object: {artifact_path.name}") + name = artifact.get("name") or artifact_path.stem + attach_dispatch_hashes(artifact, name, artifact_path, artifact_dir.parent) + artifacts[name] = artifact + expected = {"contract"} + expected.update( + artifact_name(role, platform) + for platform in PLATFORMS + for role in ["implementation", "verification"] + ) + missing = expected - set(artifacts) + if missing: + raise ValueError(f"missing role artifacts: {', '.join(sorted(missing))}") + return artifacts + + +def attach_dispatch_hashes(artifact, name, artifact_path, run_dir): + dispatch = artifact.setdefault("dispatch", {}) + dispatch.setdefault("artifact", f"role-artifacts/{name}.json") + dispatch["artifact_sha256"] = sha256_file(artifact_path) + prompt_path = resolve_run_path(run_dir, dispatch.get("prompt", f"role-prompts/{name}.md")) + if prompt_path.exists(): + actual_sha256 = sha256_file(prompt_path) + expected_sha256 = dispatch.get("prompt_sha256") + if expected_sha256 and expected_sha256 != actual_sha256: + raise ValueError(f"prompt sha256 mismatch for {name}: {dispatch.get('prompt')}") + dispatch["prompt_sha256"] = actual_sha256 + snapshot_value = dispatch.get("input_snapshot") + if snapshot_value: + snapshot_path = resolve_run_path(run_dir, snapshot_value) + if not snapshot_path.exists(): + raise ValueError(f"input snapshot does not exist for {name}: {snapshot_value}") + actual_sha256 = sha256_file(snapshot_path) + if dispatch.get("input_snapshot_sha256") != actual_sha256: + raise ValueError(f"input snapshot sha256 mismatch for {name}: {snapshot_value}") + snapshot = read_json(snapshot_path) + if not isinstance(snapshot, dict): + raise ValueError(f"input snapshot must be an object for {name}") + package_path = run_dir / "execution-package.json" + if not package_path.is_file(): + raise ValueError(f"execution package does not exist for {name}") + if snapshot.get("execution_package_sha256") != sha256_file(package_path): + raise ValueError(f"execution package sha256 mismatch for {name}") + dependency_hashes = snapshot.get("dependency_artifact_sha256", {}) + if not isinstance(dependency_hashes, dict): + raise ValueError(f"input snapshot dependencies must be an object for {name}") + for dependency, expected_sha256 in dependency_hashes.items(): + if Path(dependency).name != dependency: + raise ValueError(f"invalid dependency artifact name for {name}: {dependency}") + dependency_path = run_dir / "role-artifacts" / f"{dependency}.json" + if not dependency_path.is_file(): + raise ValueError(f"dependency artifact does not exist for {name}: {dependency}") + if sha256_file(dependency_path) != expected_sha256: + raise ValueError( + f"dependency artifact sha256 mismatch for {name}: {dependency}" + ) + command_log_value = dispatch.get("command_log") + if command_log_value: + command_log_path = resolve_run_path(run_dir, command_log_value) + if not command_log_path.exists(): + raise ValueError(f"command log does not exist for {name}: {command_log_value}") + actual_sha256 = sha256_file(command_log_path) + if dispatch.get("command_log_sha256") != actual_sha256: + raise ValueError(f"command log sha256 mismatch for {name}: {command_log_value}") + repository_delta_value = dispatch.get("repository_delta") + if repository_delta_value: + repository_delta_path = resolve_run_path(run_dir, repository_delta_value) + if not repository_delta_path.exists(): + raise ValueError( + f"repository delta does not exist for {name}: {repository_delta_value}" + ) + actual_sha256 = sha256_file(repository_delta_path) + if dispatch.get("repository_delta_sha256") != actual_sha256: + raise ValueError( + f"repository delta sha256 mismatch for {name}: {repository_delta_value}" + ) + delta = read_json(repository_delta_path) + if not isinstance(delta, dict): + raise ValueError(f"repository delta must be an object for {name}") + delta_files = { + normalize_repo_path_text(path) for path in delta.get("changed_files", []) + } + artifact_files = { + normalize_repo_path_text(path) + for path in artifact.get("output", {}).get("files_changed", []) + } + if delta_files != artifact_files: + raise ValueError(f"repository delta changed_files mismatch for {name}") + + +def strip_artifact_metadata(artifact): + return { + key: value + for key, value in artifact.items() + if key not in {"name", "role", "platform"} + } + + +def resolve_run_path(run_dir, path_text): + path = Path(path_text) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"execution artifact path must be relative to run directory: {path_text}") + return run_dir / path + + +def update_matrix(matrix_path, updates): + lines = matrix_path.read_text(encoding="utf-8").splitlines() + for update in updates: + lines = apply_matrix_update(lines, update) + matrix_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def apply_matrix_update(lines, update): + for index, line in enumerate(lines): + if not line.startswith("|"): + continue + header = split_markdown_row(line) + if "Feature" not in header or update["platform_unit"] not in header: + continue + feature_index = header.index("Feature") + platform_index = header.index(update["platform_unit"]) + for row_index in range(index + 2, len(lines)): + row_line = lines[row_index] + if not row_line.startswith("|"): + break + row = split_markdown_row(row_line) + if len(row) != len(header) or row[feature_index] != update["feature"]: + continue + status, _ = parse_matrix_cell(row[platform_index]) + if status != update["from"]: + raise ValueError( + f"matrix cell {update['feature']} / {update['platform_unit']} is {status}, expected {update['from']}" + ) + row[platform_index] = f"`{matrix_to_cell(update)}`" + lines[row_index] = "| " + " | ".join(row) + " |" + return lines + raise ValueError(f"matrix row not found for {update['feature']} / {update['platform_unit']}") + + +def matrix_to_cell(update): + cell = str(update.get("to_cell") or "").strip() + if cell.startswith("`") and cell.endswith("`"): + cell = cell[1:-1] + status, _ = parse_matrix_cell(cell) + if status != update.get("to") or any(character in cell for character in "\n\r|`"): + raise ValueError( + f"matrix to_cell must encode to={update.get('to')}: {update.get('to_cell')}" + ) + return cell + + +def artifact_name(role, platform): + return role if platform is None else f"{platform}-{role}" + + +def repo_relative_path(path): + try: + return path.resolve().relative_to(REPO_ROOT).as_posix() + except ValueError as exc: + raise ValueError(f"path must be inside the repository: {path}") from exc + + +def stable_matrix_path(path): + resolved = Path(path).resolve() + try: + return resolved.relative_to(REPO_ROOT).as_posix() + except ValueError: + return str(resolved) + + +def run_relative_path(path, run_dir): + return path.resolve().relative_to(run_dir.resolve()).as_posix() + + +def sha256_file(path): + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def sha256_text(value): + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + init_parser = subparsers.add_parser("init", help="Create a requirement workspace") + init_parser.add_argument("--matrix", required=True) + init_parser.add_argument("--feature", help="Requirement feature; defaults to highest-priority gap") + init_parser.add_argument("--sdk-family", help="Required for a feature outside the matrix backlog") + init_parser.add_argument("--key-api", action="append", help="Key SDK API; repeat as needed") + init_parser.add_argument("--target-sdk-version", required=True) + init_parser.add_argument("--run-dir", required=True) + init_parser.add_argument("--index", type=int, default=0) + init_parser.add_argument("--routing-config", default=str(DEFAULT_ROUTING_CONFIG)) + init_parser.add_argument("--repository-profile", default=str(DEFAULT_REPOSITORY_PROFILE)) + + dispatch_parser = subparsers.add_parser("dispatch", help="Dispatch one requirement phase") + dispatch_parser.add_argument("--run-dir", required=True) + dispatch_parser.add_argument("--phase", required=True, choices=PHASE_ORDER) + dispatch_parser.add_argument("--platform", action="append", choices=PLATFORMS) + dispatch_parser.add_argument("--model") + dispatch_parser.add_argument("--routing-config", default=str(DEFAULT_ROUTING_CONFIG)) + dispatch_parser.add_argument("--codex-bin", default="codex") + dispatch_parser.add_argument("--timeout-seconds", type=int, default=900) + dispatch_parser.add_argument("--retry", action="store_true") + dispatch_parser.add_argument("--dry-run", action="store_true") + + assemble_parser = subparsers.add_parser("assemble", help="Assemble the v4 manifest") + assemble_parser.add_argument("--run-dir", required=True) + assemble_parser.add_argument("--matrix", required=True) + assemble_parser.add_argument( + "--final-status", choices=["PASS", "PASS WITH RISKS", "BLOCKED"], default="BLOCKED" + ) + assemble_parser.add_argument( + "--cross-platform-result", choices=RESULT_VALUES, default="BLOCKED" + ) + assemble_parser.add_argument( + "--cross-platform-evidence", default="Pending required platform verification." + ) + assemble_parser.add_argument("--cross-platform-difference", action="append", default=[]) + args = parser.parse_args(argv) + try: + if getattr(args, "timeout_seconds", 1) <= 0: + raise ValueError("timeout-seconds must be positive") + if args.command == "init": + return init_workspace(args) + if args.command == "dispatch": + return dispatch_workspace(args) + if args.command == "assemble": + return assemble_workspace(args) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/ai-engineering/tools/orchestrate_case_execution_test.py b/docs/ai-engineering/tools/orchestrate_case_execution_test.py new file mode 100644 index 000000000..889804002 --- /dev/null +++ b/docs/ai-engineering/tools/orchestrate_case_execution_test.py @@ -0,0 +1,1138 @@ +import json +import subprocess +import sys +import tempfile +import textwrap +import time +import unittest +from pathlib import Path +from unittest import mock + + +REPO_ROOT = Path(__file__).resolve().parents[3] +ORCHESTRATOR = REPO_ROOT / "docs/ai-engineering/tools/orchestrate_case_execution.py" +VALIDATOR = REPO_ROOT / "docs/ai-engineering/tools/validate_acceptance_manifest.py" +PLATFORMS = ["android", "ios", "macos", "windows"] +TOOLS_DIR = REPO_ROOT / "docs/ai-engineering/tools" +if str(TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(TOOLS_DIR)) + +import orchestrate_case_execution as orchestrator +from orchestrate_case_execution import collect_repository_fingerprints, run_codex_role + + +class RequirementOrchestratorTest(unittest.TestCase): + TARGET_SDK_VERSION = "4.6.4" + + def write_matrix(self): + handle = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) + handle.write( + textwrap.dedent( + """ + | Feature | SDK Family | Key APIs | Android full | iOS UIKit | macOS | Windows | Notes | + | --- | --- | --- | --- | --- | --- | --- | --- | + | Join channel audio | Full RTC | `joinChannel`, `setAudioProfile` | `DONE(app/JoinChannelAudio.java)` | `DONE(APIExample/JoinChannelAudio.swift)` | `PARTIAL(APIExample/JoinChannelAudio.swift)` | `MISSING` | Keep official platforms aligned. | + + ## Confirmed Gaps + + | Gap | Affected Units | Severity | + | --- | --- | --- | + | Basic audio-only join channel | Windows | High - missing foundational case | + """ + ) + ) + handle.close() + self.addCleanup(lambda: Path(handle.name).unlink(missing_ok=True)) + return Path(handle.name) + + def run_orchestrator(self, *args): + return subprocess.run( + [sys.executable, str(ORCHESTRATOR), *args], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + def init_workspace(self, matrix_path, run_dir): + result = self.run_orchestrator( + "init", + "--matrix", + str(matrix_path), + "--feature", + "Join channel audio", + "--target-sdk-version", + self.TARGET_SDK_VERSION, + "--run-dir", + str(run_dir), + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def write_fake_codex( + self, directory, implementation_sleep=0, fail_implementation_platform=None + ): + path = Path(directory) / "fake-codex" + path.write_text( + textwrap.dedent( + f"""\ + #!/usr/bin/env python3 + import json + import sys + import time + from pathlib import Path + + if sys.argv[1:] == ["--version"]: + print("codex-cli 0.test") + raise SystemExit(0) + + args = sys.argv[1:] + output_path = Path(args[args.index("--output-last-message") + 1]) + name = output_path.stem + fail_name = {json.dumps(f'{fail_implementation_platform}-implementation' if fail_implementation_platform else '')} + if name == fail_name: + print(json.dumps({{"type": "thread.started", "thread_id": f"run-{{name}}"}}), flush=True) + raise SystemExit(7) + targets = {{ + "android": "Android/APIExample/", + "ios": "iOS/APIExample/", + "macos": "macOS/", + "windows": "windows/", + }} + build_commands = {{ + "android": "./gradlew assembleDebug", + "ios": "xcodebuild -scheme APIExample build", + "macos": "xcodebuild -scheme APIExample build", + "windows": "msbuild APIExample.sln /t:Build", + }} + status = "PASS" + if name == "contract": + output = {{ + "scenario": "Keep Join channel audio aligned across official platforms.", + "key_apis": ["joinChannel", "setAudioProfile"], + "non_goals": ["Release packaging"], + "reference": {{ + "required": True, + "source_case": "Android/APIExample/app/src/main/java/io/agora/api/example/examples/basic/JoinChannelAudio.java", + "contract_result": "PASS", + }}, + "cross_platform_requirements": ["Equivalent join and leave behavior"], + "platform_targets": {{ + platform: {{ + "required": True, + "target_project": target, + "key_constraints": [f"Keep changes in {{target}}"], + "files_allowed": [target], + "waiver_reason": "", + }} + for platform, target in targets.items() + }}, + }} + else: + platform, role = name.rsplit("-", 1) + if role == "implementation": + time.sleep({implementation_sleep}) + output = {{ + "target_project": targets[platform], + "query_cases": "No duplicate case found.", + "upsert_case": "Platform case updated.", + "files_changed": [f"{{targets[platform]}}AGENTS.md"], + "matrix_updates": [], + }} + else: + if platform == "windows" and sys.platform != "win32": + status = "BLOCKED" + output = {{ + "result": "BLOCKED", + "findings": ["Windows build requires a Windows host."], + "parity_result": "PASS", + "entry_point": "Basic > Join channel audio", + "ux_notes": "Static review only on this host.", + "commands": [], + "build_result": "BLOCKED", + "skipped_checks": [{{ + "name": "Windows MSBuild", + "reason": "Current host is not Windows.", + }}], + }} + else: + output = {{ + "result": "PASS", + "findings": ["No blocking findings."], + "parity_result": "PASS", + "entry_point": "Basic > Join channel audio", + "ux_notes": "Matches adjacent examples.", + "commands": [{{ + "kind": "build", + "command": build_commands[platform], + "result": "PASS", + }}], + "build_result": "PASS", + "skipped_checks": [], + }} + result = {{ + "status": status, + "evidence": f"{{name}} evidence", + "summary": f"{{name}} completed", + "output": output, + }} + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result), encoding="utf-8") + print(json.dumps({{"type": "thread.started", "thread_id": f"run-{{name}}"}}), flush=True) + if name != "contract" and name.endswith("-verification") and output["commands"]: + print(json.dumps({{ + "type": "item.completed", + "item": {{ + "id": f"build-{{platform}}", + "type": "command_execution", + "command": build_commands[platform], + "status": "completed", + "exit_code": 0, + }}, + }}), flush=True) + """ + ), + encoding="utf-8", + ) + path.chmod(0o755) + return path + + def dispatch_contract(self, run_dir, codex_bin): + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "contract", + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + + def test_init_creates_shared_contract_and_eight_platform_role_tasks(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + + manifest = json.loads((run_dir / "acceptance-manifest.json").read_text()) + package = json.loads((run_dir / "execution-package.json").read_text()) + artifact_names = sorted(path.stem for path in (run_dir / "role-artifacts").glob("*.json")) + expected = ["contract"] + [ + f"{platform}-{role}" + for platform in PLATFORMS + for role in ["implementation", "verification"] + ] + self.assertEqual(artifact_names, sorted(expected)) + self.assertEqual(sorted(manifest["platforms"]), PLATFORMS) + self.assertNotIn("roles", manifest) + self.assertEqual( + package["repository_profile"], + "docs/ai-engineering/repository-profile.json", + ) + self.assertEqual(len(package["repository_profile_sha256"]), 64) + + def test_execution_configuration_rejects_repository_profile_drift(self): + routing_path = REPO_ROOT / "docs/ai-engineering/role-routing.json" + package = { + "routing_config": "docs/ai-engineering/role-routing.json", + "routing_config_sha256": orchestrator.sha256_file(routing_path), + "repository_profile": "docs/ai-engineering/repository-profile.json", + "repository_profile_sha256": "0" * 64, + } + + with self.assertRaisesRegex(ValueError, "repository profile content changed"): + orchestrator.validate_execution_configuration(package, routing_path) + + def test_contract_input_snapshot_binds_repository_profile(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + + package = json.loads((run_dir / "execution-package.json").read_text()) + artifact = json.loads((run_dir / "role-artifacts/contract.json").read_text()) + snapshot_path = run_dir / artifact["dispatch"]["input_snapshot"] + snapshot = json.loads(snapshot_path.read_text()) + + self.assertEqual(snapshot["repository_profile"], package["repository_profile"]) + self.assertEqual( + snapshot["repository_profile_sha256"], + package["repository_profile_sha256"], + ) + + def test_init_accepts_new_requirement_outside_the_matrix_backlog(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + result = self.run_orchestrator( + "init", + "--matrix", + str(matrix_path), + "--feature", + "Spatial audio", + "--sdk-family", + "Full RTC", + "--key-api", + "enableSpatialAudio", + "--target-sdk-version", + self.TARGET_SDK_VERSION, + "--run-dir", + str(run_dir), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + package = json.loads((run_dir / "execution-package.json").read_text()) + self.assertEqual(package["requirement"]["feature"], "Spatial audio") + self.assertEqual(package["requirement"]["key_apis"], ["enableSpatialAudio"]) + contract_prompt = (run_dir / "role-prompts/contract.md").read_text() + self.assertNotIn("If Contract marks this platform required=false", contract_prompt) + + def test_repository_matrix_identity_is_portable(self): + identity = orchestrator.stable_matrix_path( + REPO_ROOT / "docs/ai-engineering/case-maintenance-matrix.md" + ) + + self.assertEqual(identity, "docs/ai-engineering/case-maintenance-matrix.md") + + def test_platform_implementation_waits_for_shared_contract(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "implementation", + "--platform", + "android", + "--model", + "test-model", + "--dry-run", + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("Contract gate returned status BLOCKED", result.stderr) + + def test_contract_semantic_failure_is_blocked_before_implementation(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + artifact_path = run_dir / "role-artifacts/contract.json" + artifact = json.loads(artifact_path.read_text()) + artifact["status"] = "PASS" + artifact["output"]["reference"]["contract_result"] = "PASS" + artifact["output"]["cross_platform_requirements"] = [] + artifact_path.write_text(json.dumps(artifact), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "Contract semantic gate failed"): + orchestrator.validate_completed_contract(run_dir) + + blocked = json.loads(artifact_path.read_text()) + self.assertEqual(blocked["status"], "BLOCKED") + self.assertIn("cross_platform_requirements", blocked["evidence"]) + + def test_implementation_phase_dispatches_four_platform_agents_with_scoped_deltas(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "implementation", + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + ) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + for platform in PLATFORMS: + artifact = json.loads( + (run_dir / f"role-artifacts/{platform}-implementation.json").read_text() + ) + self.assertEqual(artifact["dispatch"]["mode"], "codex-exec") + self.assertEqual(artifact["dispatch"]["run_id"], f"run-{platform}-implementation") + self.assertEqual(artifact["output"]["files_changed"], []) + + def test_implementation_reconciles_each_agent_before_starting_the_next(self): + tasks = [ + {"name": "android-implementation", "platform": "android"}, + {"name": "ios-implementation", "platform": "ios"}, + ] + events = [] + + with mock.patch.object( + orchestrator, + "refresh_implementation_input_snapshot", + side_effect=lambda run_dir, task: events.append(f"snapshot:{task['name']}"), + ), mock.patch.object( + orchestrator, + "collect_repository_fingerprints", + side_effect=[{"state": "android"}, {"state": "ios"}], + ), mock.patch.object( + orchestrator, + "run_codex_role", + side_effect=lambda **kwargs: events.append(f"run:{kwargs['name']}"), + ), mock.patch.object( + orchestrator, + "reconcile_implementation_changes", + side_effect=lambda run_dir, task_group, before: events.append( + f"reconcile:{task_group[0]['name']}" + ), + ), mock.patch.object( + orchestrator, + "validate_completed_implementation", + side_effect=lambda run_dir, platform: events.append(f"validate:{platform}"), + ): + orchestrator.run_implementation_tasks(Path("/tmp/run"), tasks) + + self.assertEqual( + events, + [ + "snapshot:android-implementation", + "run:android-implementation", + "reconcile:android-implementation", + "validate:android", + "snapshot:ios-implementation", + "run:ios-implementation", + "reconcile:ios-implementation", + "validate:ios", + ], + ) + + def test_implementation_semantic_failure_blocks_verification(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + artifact_path = run_dir / "role-artifacts/android-implementation.json" + artifact = json.loads(artifact_path.read_text()) + artifact["status"] = "PASS" + artifact["output"]["query_cases"] = "" + artifact_path.write_text(json.dumps(artifact), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "Implementation semantic gate failed"): + orchestrator.validate_completed_implementation(run_dir, "android") + + blocked = json.loads(artifact_path.read_text()) + self.assertEqual(blocked["status"], "BLOCKED") + self.assertIn("query_cases", blocked["evidence"]) + + def test_implementation_refreshes_snapshot_after_previous_platform_delta(self): + task = { + "name": "ios-implementation", + "snapshot_dependencies": {"contract": {}}, + "resolved_routing": {"routing_config": "routing.json"}, + "input_snapshot": "old.json", + "input_snapshot_sha256": "old-sha", + "expected_repository_state": {"head": "old"}, + } + snapshot_path = Path("/tmp/run/input-snapshots/ios-new.json") + + with mock.patch.object( + orchestrator, + "write_input_snapshot", + return_value=(snapshot_path, "new-sha"), + ), mock.patch.object( + orchestrator, + "read_json", + return_value={"repository_state": {"head": "after-android"}}, + ): + orchestrator.refresh_implementation_input_snapshot(Path("/tmp/run"), task) + + self.assertEqual(task["input_snapshot"], "input-snapshots/ios-new.json") + self.assertEqual(task["input_snapshot_sha256"], "new-sha") + self.assertEqual(task["expected_repository_state"], {"head": "after-android"}) + + def test_repository_fingerprints_include_untracked_file_contents(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + path = root / "new-example.swift" + path.write_text("first\n", encoding="utf-8") + before = collect_repository_fingerprints(root) + + path.write_text("second\n", encoding="utf-8") + after = collect_repository_fingerprints(root) + + self.assertNotEqual(before["new-example.swift"], after["new-example.swift"]) + + def test_artifact_loading_rejects_stale_dependency_snapshot(self): + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + artifact_dir = run_dir / "role-artifacts" + artifact_dir.mkdir(parents=True) + package_path = run_dir / "execution-package.json" + package_path.write_text('{"requirement":{}}\n', encoding="utf-8") + contract_path = artifact_dir / "contract.json" + contract_path.write_text('{"status":"PASS"}\n', encoding="utf-8") + snapshot_path = run_dir / "input-snapshots/android-verification.json" + snapshot_path.parent.mkdir(parents=True) + snapshot_path.write_text( + json.dumps( + { + "execution_package_sha256": orchestrator.sha256_file(package_path), + "dependency_artifact_sha256": {"contract": "0" * 64}, + } + ), + encoding="utf-8", + ) + artifact_path = artifact_dir / "android-verification.json" + artifact = { + "dispatch": { + "mode": "codex-exec", + "input_snapshot": "input-snapshots/android-verification.json", + "input_snapshot_sha256": orchestrator.sha256_file(snapshot_path), + } + } + artifact_path.write_text(json.dumps(artifact), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "dependency artifact sha256 mismatch"): + orchestrator.attach_dispatch_hashes( + artifact, + "android-verification", + artifact_path, + run_dir, + ) + + def test_implementation_retry_delta_covers_all_attempt_changes(self): + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + contract_path = run_dir / "role-artifacts/contract.json" + contract_path.parent.mkdir(parents=True) + contract_path.write_text( + json.dumps( + { + "output": { + "platform_targets": { + "android": {"files_allowed": ["Android/APIExample/"]} + } + } + } + ), + encoding="utf-8", + ) + artifact_path = run_dir / "role-artifacts/android-implementation.json" + artifact = { + "status": "BLOCKED", + "dispatch": { + "mode": "codex-exec", + "run_id": "run-1", + "input_snapshot_sha256": "snapshot-1", + }, + "output": {"files_changed": []}, + } + artifact_path.write_text(json.dumps(artifact), encoding="utf-8") + first_before = {"Android/APIExample/A.java": {"sha256": "before-a"}} + first_after = {"Android/APIExample/A.java": {"sha256": "after-a"}} + first_task = { + "name": "android-implementation", + "platform": "android", + "input_snapshot": "input-snapshots/android-1.json", + "input_snapshot_sha256": "snapshot-1", + "previous_files_changed": [], + } + with mock.patch.object( + orchestrator, "collect_repository_fingerprints", return_value=first_after + ): + orchestrator.reconcile_implementation_changes( + run_dir, [first_task], first_before + ) + + artifact = json.loads(artifact_path.read_text()) + artifact["status"] = "PASS" + artifact["dispatch"]["run_id"] = "run-2" + artifact["dispatch"]["input_snapshot_sha256"] = "snapshot-2" + artifact_path.write_text(json.dumps(artifact), encoding="utf-8") + second_after = { + **first_after, + "Android/APIExample/B.java": {"sha256": "after-b"}, + } + second_task = { + "name": "android-implementation", + "platform": "android", + "input_snapshot": "input-snapshots/android-2.json", + "input_snapshot_sha256": "snapshot-2", + "previous_files_changed": ["Android/APIExample/A.java"], + } + with mock.patch.object( + orchestrator, "collect_repository_fingerprints", return_value=second_after + ): + orchestrator.reconcile_implementation_changes( + run_dir, [second_task], first_after + ) + + artifact = json.loads(artifact_path.read_text()) + delta = json.loads( + (run_dir / "repository-deltas/android-implementation.json").read_text() + ) + expected = ["Android/APIExample/A.java", "Android/APIExample/B.java"] + self.assertEqual(artifact["output"]["files_changed"], expected) + self.assertEqual(delta["changed_files"], expected) + self.assertEqual([attempt["run_id"] for attempt in delta["attempts"]], ["run-1", "run-2"]) + + def test_failed_implementation_still_reconciles_successful_artifacts(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex( + tmpdir, fail_implementation_platform="windows" + ) + self.dispatch_contract(run_dir, codex_bin) + + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "implementation", + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + ) + + self.assertEqual(result.returncode, 1) + android = json.loads( + (run_dir / "role-artifacts/android-implementation.json").read_text() + ) + self.assertEqual(android["output"]["files_changed"], []) + self.assertIn("repository_delta", android["dispatch"]) + + def test_failed_verification_mutation_creates_persistent_scope_violation(self): + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + prompt_path = run_dir / "role-prompts/ios-verification.md" + prompt_path.parent.mkdir(parents=True) + prompt_path.write_text("verify", encoding="utf-8") + artifact_path = run_dir / "role-artifacts/ios-verification.json" + + with mock.patch( + "orchestrate_case_execution.repository_state", + return_value={"state": "after"}, + ): + with self.assertRaisesRegex(ValueError, "modified repository files"): + run_codex_role( + command=[sys.executable, "-c", "raise SystemExit(7)"], + name="ios-verification", + role="verification", + platform="ios", + prompt_path=prompt_path, + artifact_path=artifact_path, + raw_result_path=run_dir / "raw-results/ios-verification.json", + log_path=run_dir / "dispatch-logs/ios-verification.jsonl", + stderr_path=run_dir / "dispatch-logs/ios-verification.stderr.log", + model_profile="review", + model="test-model", + reasoning_effort="high", + codex_version="codex-cli test", + working_directory=Path(tmpdir), + working_directory_manifest="iOS/APIExample/", + input_snapshot="input-snapshots/ios-verification.json", + input_snapshot_sha256="a" * 64, + expected_repository_state={"state": "before"}, + timeout_seconds=5, + ) + + self.assertTrue( + (run_dir / "repository-deltas/ios-verification-scope-violation.json").exists() + ) + + def test_macos_rejects_windows_build_pass(self): + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + prompt_path = run_dir / "role-prompts/windows-verification.md" + prompt_path.parent.mkdir(parents=True) + prompt_path.write_text("verify", encoding="utf-8") + raw_result_path = run_dir / "raw-results/windows-verification.json" + raw_result_path.parent.mkdir(parents=True) + raw_result_path.write_text( + json.dumps( + { + "status": "PASS", + "evidence": "MSBuild passed.", + "summary": "Windows verified.", + "output": { + "result": "PASS", + "findings": [], + "parity_result": "PASS", + "entry_point": "Basic > Join channel audio", + "ux_notes": "Matches adjacent examples.", + "commands": [ + { + "kind": "build", + "command": "msbuild APIExample.sln /t:Build", + "result": "PASS", + } + ], + "build_result": "PASS", + "skipped_checks": [], + }, + } + ), + encoding="utf-8", + ) + command = [ + sys.executable, + "-c", + ( + "import json; " + "print(json.dumps({'type':'thread.started','thread_id':'run-windows'})); " + "print(json.dumps({'type':'item.completed','item':" + "{'id':'build','type':'command_execution','command':" + "'msbuild APIExample.sln /t:Build','exit_code':0}}))" + ), + ] + + with mock.patch.object(orchestrator.sys, "platform", "darwin"), mock.patch.object( + orchestrator, + "repository_state", + return_value={"state": "clean"}, + ): + with self.assertRaisesRegex(ValueError, "requires a Windows host"): + run_codex_role( + command=command, + name="windows-verification", + role="verification", + platform="windows", + prompt_path=prompt_path, + artifact_path=run_dir / "role-artifacts/windows-verification.json", + raw_result_path=raw_result_path, + log_path=run_dir / "dispatch-logs/windows-verification.jsonl", + stderr_path=run_dir / "dispatch-logs/windows-verification.stderr.log", + model_profile="review", + model="test-model", + reasoning_effort="high", + codex_version="codex-cli test", + working_directory=Path(tmpdir), + working_directory_manifest="windows/", + input_snapshot="input-snapshots/windows-verification.json", + input_snapshot_sha256="a" * 64, + expected_repository_state={"state": "clean"}, + timeout_seconds=5, + ) + + def test_verification_semantic_failure_becomes_retryable(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + artifact_path = run_dir / "role-artifacts/ios-verification.json" + artifact = json.loads(artifact_path.read_text()) + artifact["status"] = "PASS" + artifact["output"]["result"] = "BLOCKED" + artifact_path.write_text(json.dumps(artifact), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "Verification semantic gate failed"): + orchestrator.validate_completed_verification(run_dir, "ios") + + blocked = json.loads(artifact_path.read_text()) + self.assertEqual(blocked["status"], "BLOCKED") + self.assertIn("output.result", blocked["evidence"]) + + def test_platform_filter_dispatches_only_requested_agent(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "implementation", + "--platform", + "windows", + "--model", + "test-model", + "--dry-run", + ) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + payload = json.loads(result.stdout) + self.assertEqual([item["platform"] for item in payload["commands"]], ["windows"]) + command = payload["commands"][0]["command"] + self.assertIn("-C", command) + self.assertIn(str(REPO_ROOT / "windows"), command) + + def test_platform_prompt_uses_target_selected_by_completed_contract(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + contract_path = run_dir / "role-artifacts/contract.json" + contract = json.loads(contract_path.read_text()) + contract["output"]["platform_targets"]["ios"]["target_project"] = "iOS/APIExample-OC/" + contract["output"]["platform_targets"]["ios"]["files_allowed"] = ["iOS/APIExample-OC/"] + contract_path.write_text(json.dumps(contract), encoding="utf-8") + + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "implementation", + "--platform", + "ios", + "--model", + "test-model", + "--dry-run", + ) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + prompt = (run_dir / "role-prompts/ios-implementation.md").read_text() + self.assertIn("Target: iOS/APIExample-OC/", prompt) + self.assertIn("If Contract marks this platform required=false", prompt) + + def test_verification_uses_corresponding_platform_implementation(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + implementation = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "implementation", + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + ) + self.assertEqual(implementation.returncode, 0, implementation.stderr) + + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "verification", + "--platform", + "windows", + "--model", + "test-model", + "--dry-run", + ) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + prompt = (run_dir / "role-prompts/windows-verification.md").read_text() + self.assertIn("windows-implementation completed", prompt) + self.assertIn("macOS cannot provide Windows build evidence", prompt) + self.assertIn("Do not download Windows SDK archives", prompt) + implementation_prompt = (run_dir / "role-prompts/windows-implementation.md").read_text() + self.assertNotIn("cannot provide Windows build evidence", implementation_prompt) + + def test_platform_dispatch_timeout_keeps_other_platform_scope_isolated(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir, implementation_sleep=2) + self.dispatch_contract(run_dir, codex_bin) + + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "implementation", + "--platform", + "android", + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + "--timeout-seconds", + "1", + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("android-implementation timed out after 1 seconds", result.stderr) + self.assertTrue((run_dir / "dispatch-logs/android-implementation.jsonl").exists()) + self.assertFalse((run_dir / "dispatch-logs/ios-implementation.jsonl").exists()) + + def test_timeout_terminates_agent_child_processes(self): + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + prompt_path = run_dir / "role-prompts/android-implementation.md" + prompt_path.parent.mkdir(parents=True) + prompt_path.write_text("implement", encoding="utf-8") + marker = Path(tmpdir) / "late-child-write.txt" + child_code = ( + "import time; from pathlib import Path; " + f"time.sleep(0.4); Path({str(marker)!r}).write_text('late')" + ) + parent_code = ( + "import subprocess, sys, time; " + f"subprocess.Popen([sys.executable, '-c', {child_code!r}]); " + "time.sleep(10)" + ) + + with self.assertRaisesRegex(ValueError, "timed out"): + run_codex_role( + command=[sys.executable, "-c", parent_code], + name="android-implementation", + role="implementation", + platform="android", + prompt_path=prompt_path, + artifact_path=run_dir / "role-artifacts/android-implementation.json", + raw_result_path=run_dir / "raw-results/android-implementation.json", + log_path=run_dir / "dispatch-logs/android-implementation.jsonl", + stderr_path=run_dir / "dispatch-logs/android-implementation.stderr.log", + model_profile="deep", + model="test-model", + reasoning_effort="high", + codex_version="codex-cli test", + working_directory=Path(tmpdir), + working_directory_manifest="Android/APIExample/", + input_snapshot="input-snapshots/android-implementation.json", + input_snapshot_sha256="a" * 64, + expected_repository_state={"state": "clean"}, + timeout_seconds=0.1, + ) + time.sleep(0.6) + + self.assertFalse(marker.exists()) + + def test_full_multi_platform_run_assembles_blocked_v4_manifest(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + for phase in ["implementation", "verification"]: + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + phase, + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + + result = self.run_orchestrator( + "assemble", + "--run-dir", + str(run_dir), + "--matrix", + str(matrix_path), + "--final-status", + "BLOCKED", + "--cross-platform-result", + "BLOCKED", + "--cross-platform-evidence", + "Target SDK and Windows-host build evidence are pending.", + ) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + final_path = run_dir / "final-acceptance-manifest.json" + final_manifest = json.loads(final_path.read_text()) + self.assertEqual(final_manifest["version"], 4) + self.assertEqual(final_manifest["cross_platform_acceptance"]["result"], "BLOCKED") + self.assertEqual(sorted(final_manifest["platforms"]), PLATFORMS) + self.assertTrue(final_manifest["release"]["required"]) + self.assertEqual(final_manifest["release"]["target_sdk_version"], "4.6.4") + validation = subprocess.run( + [sys.executable, str(VALIDATOR), str(final_path)], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(validation.returncode, 0, validation.stderr) + + def test_assemble_reports_only_repository_gates_without_external_metadata(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + for phase in ["implementation", "verification"]: + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + phase, + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + + result = self.run_orchestrator( + "assemble", + "--run-dir", + str(run_dir), + "--matrix", + str(matrix_path), + "--final-status", + "PASS", + "--cross-platform-result", + "PASS", + "--cross-platform-evidence", + "All required platforms passed independent verification.", + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("sdk-version-android", result.stderr) + self.assertNotIn("qa_acceptance", result.stderr) + + def test_assemble_rejects_matrix_path_changed_since_init(self): + matrix_path = self.write_matrix() + other_matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + + result = self.run_orchestrator( + "assemble", + "--run-dir", + str(run_dir), + "--matrix", + str(other_matrix_path), + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("matrix path changed since init", result.stderr) + + def test_assemble_rejects_requirement_changed_since_init(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + manifest_path = run_dir / "acceptance-manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["requirement"]["feature"] = "Different requirement" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + result = self.run_orchestrator( + "assemble", + "--run-dir", + str(run_dir), + "--matrix", + str(matrix_path), + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("requirement changed since init", result.stderr) + + def test_assemble_rejects_missing_completed_prompt_evidence(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + (run_dir / "role-prompts/contract.md").unlink() + + result = self.run_orchestrator( + "assemble", + "--run-dir", + str(run_dir), + "--matrix", + str(matrix_path), + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("contract.dispatch.prompt does not exist", result.stderr) + + def test_retrying_platform_implementation_invalidates_old_verification(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + self.init_workspace(matrix_path, run_dir) + codex_bin = self.write_fake_codex(tmpdir) + self.dispatch_contract(run_dir, codex_bin) + for phase in ["implementation", "verification"]: + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + phase, + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + implementation_path = run_dir / "role-artifacts/windows-implementation.json" + implementation = json.loads(implementation_path.read_text()) + implementation["status"] = "BLOCKED" + implementation_path.write_text(json.dumps(implementation), encoding="utf-8") + + result = self.run_orchestrator( + "dispatch", + "--run-dir", + str(run_dir), + "--phase", + "implementation", + "--platform", + "windows", + "--retry", + "--model", + "test-model", + "--codex-bin", + str(codex_bin), + ) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + verification = json.loads( + (run_dir / "role-artifacts/windows-verification.json").read_text() + ) + self.assertEqual(verification["status"], "BLOCKED") + self.assertEqual(verification["dispatch"]["mode"], "pending") + + def test_removed_single_platform_phases_are_rejected(self): + result = self.run_orchestrator( + "dispatch", + "--run-dir", + "/tmp/not-used", + "--phase", + "target-test", + "--model", + "test-model", + "--dry-run", + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid choice", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ai-engineering/tools/prepare_case_execution.py b/docs/ai-engineering/tools/prepare_case_execution.py new file mode 100644 index 000000000..dc729a7b2 --- /dev/null +++ b/docs/ai-engineering/tools/prepare_case_execution.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Prepare one cross-platform API Examples requirement package.""" + +import argparse +import json +import re +import sys +from pathlib import Path + +from generate_case_backlog import DEFAULT_MATRIX, REPO_ROOT, generate_execution_units + + +PLATFORMS = ["android", "ios", "macos", "windows"] +DEFAULT_REPOSITORY_PROFILE = REPO_ROOT / "docs/ai-engineering/repository-profile.json" +PROFILE_SOURCE_FIELDS = { + "gradle-property": {"path", "kind", "key"}, + "cocoapods": {"path", "kind", "package"}, + "archive-name": {"path", "kind", "prefix", "suffix"}, +} +SEMVER_CAPTURE = r"([0-9]+\.[0-9]+\.[0-9]+)" +DEFAULT_PLATFORM_TARGETS = { + "android": "Android/APIExample/", + "ios": "iOS/APIExample/", + "macos": "macOS/", + "windows": "windows/", +} +PLATFORM_UNIT_GROUPS = { + "Android full": "android", + "Android audio": "android", + "Android Compose": "android", + "iOS UIKit": "ios", + "iOS SwiftUI": "ios", + "iOS Objective-C": "ios", + "iOS audio": "ios", + "macOS": "macos", + "Windows": "windows", +} +ROLE_CONTRACTS = { + "contract": [ + "shared scenario, APIs, and non-goals", + "cross-platform behavior contract", + "target project and allowed files for every platform", + "reference contract when required", + ], + "implementation": [ + "target project", + "query-cases and upsert-case results", + "changed files", + "matrix update proposals", + ], + "verification": [ + "findings-first review and parity result", + "entry point and UX consistency", + "exact target command strings and command kinds for JSONL evidence binding", + "target build commands and result", + "skipped checks with reasons", + ], +} + + +def select_execution_unit(units, feature=None, index=0): + filtered = [unit for unit in units if feature is None or unit["feature"] == feature] + if not filtered: + suffix = f" for feature={feature}" if feature else "" + raise ValueError(f"no execution units found{suffix}") + if index < 0 or index >= len(filtered): + raise ValueError(f"execution unit index {index} is out of range; {len(filtered)} unit(s) available") + return filtered[index] + + +def source_case_from_candidate(candidate): + if not candidate: + return "" + return f"{candidate['project'].rstrip('/')}/{candidate['path'].lstrip('/')}" + + +def resolve_source_case(candidate): + source_case = source_case_from_candidate(candidate) + if not source_case: + return "" + direct_path = REPO_ROOT / source_case + if direct_path.exists(): + return source_case + project_root = REPO_ROOT / candidate["project"] + basename = Path(candidate["path"].rstrip("/")).name + if not project_root.exists() or not basename: + return source_case + matches = [ + path + for path in project_root.rglob(basename) + if "build" not in path.relative_to(project_root).parts + ] + if not matches: + return source_case + matches.sort(key=lambda path: (len(path.parts), path.as_posix())) + return matches[0].relative_to(REPO_ROOT).as_posix() + + +def pending_dispatch(name): + return { + "mode": "pending", + "prompt": f"role-prompts/{name}.md", + "artifact": f"role-artifacts/{name}.json", + } + + +def pending_artifact(name, summary, output): + return { + "agent_id": f"{name}-agent-pending", + "dispatch": pending_dispatch(name), + "status": "BLOCKED", + "evidence": f"Pending {name} gate.", + "summary": summary, + "output": output, + } + + +def default_platform_targets(): + return { + platform: { + "required": True, + "target_project": target, + "key_constraints": [ + f"Use the {platform} project rules and keep source changes inside {target}" + ], + "files_allowed": [target], + "waiver_reason": "", + } + for platform, target in DEFAULT_PLATFORM_TARGETS.items() + } + + +def load_repository_profile(profile_path=DEFAULT_REPOSITORY_PROFILE): + path = Path(profile_path) + profile = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(profile, dict): + raise ValueError("repository profile must be a JSON object") + if set(profile) != {"version", "sdk_version_sources"}: + raise ValueError("repository profile must contain only version and sdk_version_sources") + if profile.get("version") != 1: + raise ValueError("repository profile version must be 1") + sources = profile.get("sdk_version_sources") + if not isinstance(sources, dict) or set(sources) != set(PLATFORMS): + raise ValueError("repository profile must define SDK version sources for every platform") + for platform in PLATFORMS: + entries = sources[platform] + if not isinstance(entries, list) or not entries: + raise ValueError(f"repository profile {platform} SDK version sources must be non-empty") + seen_paths = set() + for index, source in enumerate(entries): + if not isinstance(source, dict): + raise ValueError(f"repository profile {platform} source {index} must be an object") + kind = source.get("kind") + expected_fields = PROFILE_SOURCE_FIELDS.get(kind) + if expected_fields is None: + raise ValueError( + f"repository profile {platform} source {index} has unknown kind {kind}" + ) + if set(source) != expected_fields: + raise ValueError( + f"repository profile {platform} source {index} must contain " + + ", ".join(sorted(expected_fields)) + ) + if not all( + isinstance(source[field], str) and source[field] for field in expected_fields + ): + raise ValueError( + f"repository profile {platform} source {index} fields must be non-empty strings" + ) + source_path = Path(source["path"]) + if source_path.is_absolute() or ".." in source_path.parts: + raise ValueError( + "repository profile source path must be repository-relative: " + + source["path"] + ) + normalized_path = source_path.as_posix() + if normalized_path in seen_paths: + raise ValueError( + f"repository profile contains duplicate source path: {normalized_path}" + ) + seen_paths.add(normalized_path) + return profile + + +def sdk_version_pattern(source): + kind = source["kind"] + if kind == "gradle-property": + return rf"(?m)^\s*{re.escape(source['key'])}\s*=\s*{SEMVER_CAPTURE}\s*$" + if kind == "cocoapods": + package = re.escape(source["package"]) + return rf"pod\s+['\"]{package}['\"]\s*,\s*['\"]{SEMVER_CAPTURE}['\"]" + if kind == "archive-name": + return re.escape(source["prefix"]) + SEMVER_CAPTURE + re.escape(source["suffix"]) + raise ValueError(f"unsupported SDK version source kind: {kind}") + + +def collect_sdk_version_checks( + target_sdk_version, + repo_root=REPO_ROOT, + sources=None, + profile_path=DEFAULT_REPOSITORY_PROFILE, +): + if sources is None: + sources = load_repository_profile(profile_path)["sdk_version_sources"] + checks = [] + for platform in PLATFORMS: + entries = sources[platform] + actual_versions = {} + problems = [] + for source in entries: + path_text = source["path"] + path = Path(repo_root) / path_text + if not path.exists(): + actual_versions[path_text] = "" + problems.append(f"missing {path_text}") + continue + pattern = sdk_version_pattern(source) + matches = sorted(set(re.findall(pattern, path.read_text(encoding="utf-8")))) + if len(matches) != 1: + actual_versions[path_text] = "" + problems.append(f"expected one SDK version in {path_text}, found {len(matches)}") + continue + actual_versions[path_text] = matches[0] + if matches[0] != target_sdk_version: + problems.append(f"{path_text}={matches[0]}") + result = "PASS" if not problems else "BLOCKED" + checks.append( + { + "name": f"sdk-version-{platform}", + "result": result, + "expected_version": target_sdk_version, + "actual_versions": actual_versions, + "evidence": ( + "; ".join(f"{path}={version}" for path, version in actual_versions.items()) + if actual_versions + else "" + ), + "reason": "" if not problems else f"Expected {target_sdk_version}; " + "; ".join(problems), + } + ) + return checks + + +def build_manifest_seed(requirement, source_case, version_sources): + reference_required = bool(source_case) + reference_result = "BLOCKED" if reference_required else "SKIPPED" + targets = default_platform_targets() + contract = pending_artifact( + "contract", + "Pending shared product, architecture, and reference contract.", + { + "scenario": f"Implement {requirement['feature']} consistently across official platforms.", + "key_apis": requirement["key_apis"], + "non_goals": [], + "reference": { + "required": reference_required, + "source_case": source_case, + "contract_result": reference_result, + }, + "cross_platform_requirements": [], + "platform_targets": targets, + }, + ) + platforms = {} + for platform, target in DEFAULT_PLATFORM_TARGETS.items(): + platforms[platform] = { + "implementation": pending_artifact( + f"{platform}-implementation", + f"Pending {platform} implementation.", + { + "target_project": target, + "query_cases": "Pending target project case query.", + "upsert_case": "Pending target project update.", + "files_changed": [], + "matrix_updates": [], + }, + ), + "verification": pending_artifact( + f"{platform}-verification", + f"Pending independent {platform} verification.", + { + "result": "BLOCKED", + "findings": [], + "parity_result": reference_result, + "entry_point": f"Pending {platform} entry point verification.", + "ux_notes": "Pending verification.", + "commands": [], + "build_result": "BLOCKED", + "skipped_checks": [], + }, + ), + } + return { + "version": 4, + "final_status": "BLOCKED", + "requirement": requirement, + "contract": contract, + "platforms": platforms, + "cross_platform_acceptance": { + "result": "BLOCKED", + "evidence": "Pending required platform verification.", + "differences": [], + }, + "release": { + "required": True, + "target_sdk_version": requirement["target_sdk_version"], + "checks": collect_sdk_version_checks( + requirement["target_sdk_version"], sources=version_sources + ), + "skipped_checks": [], + }, + "knowledge_updates": [], + } + + +def build_platform_units(units, feature): + result = { + platform: { + "target_project": DEFAULT_PLATFORM_TARGETS[platform], + "matrix_candidates": [], + } + for platform in PLATFORMS + } + for unit in units: + if unit["feature"] != feature: + continue + platform = PLATFORM_UNIT_GROUPS.get(unit["platform_unit"]) + if platform: + result[platform]["matrix_candidates"].append(unit) + return result + + +def prepare_case_execution( + matrix_path, + feature=None, + index=0, + sdk_family=None, + key_apis=None, + target_sdk_version=None, + repository_profile=DEFAULT_REPOSITORY_PROFILE, +): + if not target_sdk_version: + raise ValueError("target_sdk_version is required") + profile = load_repository_profile(repository_profile) + backlog = generate_execution_units(matrix_path) + matching = [ + unit for unit in backlog["execution_units"] if feature is None or unit["feature"] == feature + ] + selected = select_execution_unit(matching, None, index) if matching else None + if selected is None and not (feature and sdk_family and key_apis): + raise ValueError( + "a requirement outside the actionable matrix needs feature, sdk_family, and key_apis" + ) + requirement = { + "feature": feature or selected["feature"], + "sdk_family": sdk_family or selected["sdk_family"], + "key_apis": key_apis or selected["key_apis"], + "target_sdk_version": target_sdk_version, + } + source_candidate = None + if selected and selected["reference_candidates"]: + source_candidate = selected["reference_candidates"][0] + source_case = resolve_source_case(source_candidate) + blockers = [] + if not source_case: + blockers.append("No DONE reference candidate found; Contract must identify a source case.") + return { + "requirement": requirement, + "platform_units": build_platform_units(backlog["execution_units"], requirement["feature"]), + "reference_contract": { + "required": bool(source_case), + "source_candidate": source_candidate, + "source_case": source_case, + }, + "role_contracts": ROLE_CONTRACTS, + "execution_steps": [ + "Run one shared Contract for all official platforms.", + "Run Android, iOS, macOS, and Windows Implementation agents independently with attributed deltas.", + "Run independent platform Verification agents in parallel.", + "Record cross-platform differences and final acceptance.", + "Validate the manifest before applying matrix updates.", + ], + "acceptance_manifest_seed": build_manifest_seed( + requirement, source_case, profile["sdk_version_sources"] + ), + "validation_command": "python3 docs/ai-engineering/tools/validate_acceptance_manifest.py ", + "blockers": blockers, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--matrix", default=str(DEFAULT_MATRIX)) + parser.add_argument("--feature", help="Requirement feature; defaults to the highest-priority gap") + parser.add_argument("--sdk-family", help="Required when the feature is not actionable in the matrix") + parser.add_argument("--key-api", action="append", help="Key SDK API; repeat for multiple APIs") + parser.add_argument("--target-sdk-version", required=True) + parser.add_argument("--repository-profile", default=str(DEFAULT_REPOSITORY_PROFILE)) + parser.add_argument("--index", type=int, default=0) + parser.add_argument("--output") + args = parser.parse_args(argv) + try: + package = prepare_case_execution( + Path(args.matrix), + args.feature, + index=args.index, + sdk_family=args.sdk_family, + key_apis=args.key_api, + target_sdk_version=args.target_sdk_version, + repository_profile=Path(args.repository_profile), + ) + except (OSError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + payload = json.dumps(package, indent=2, ensure_ascii=False) + "\n" + if args.output: + Path(args.output).write_text(payload, encoding="utf-8") + else: + print(payload, end="") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/ai-engineering/tools/prepare_case_execution_test.py b/docs/ai-engineering/tools/prepare_case_execution_test.py new file mode 100644 index 000000000..3ee33b3d5 --- /dev/null +++ b/docs/ai-engineering/tools/prepare_case_execution_test.py @@ -0,0 +1,212 @@ +import json +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + + +TOOLS_DIR = Path(__file__).resolve().parent +if str(TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(TOOLS_DIR)) + +from prepare_case_execution import ( + collect_sdk_version_checks, + load_repository_profile, + prepare_case_execution, +) +from validate_acceptance_manifest import validate_manifest + + +PLATFORMS = ["android", "ios", "macos", "windows"] + + +class PrepareCaseExecutionTest(unittest.TestCase): + TARGET_SDK_VERSION = "4.6.4" + + def write_matrix(self): + handle = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) + handle.write( + textwrap.dedent( + """ + | Feature | SDK Family | Key APIs | Android full | iOS UIKit | macOS | Windows | Notes | + | --- | --- | --- | --- | --- | --- | --- | --- | + | Join channel audio | Full RTC | `joinChannel`, `setAudioProfile` | `DONE(app/JoinChannelAudio.java)` | `DONE/APIExample/JoinChannelAudio.swift` | `PARTIAL(APIExample/JoinChannelAudio.swift)` | `MISSING` | Keep all official platforms aligned. | + + ## Confirmed Gaps + + | Gap | Affected Units | Severity | + | --- | --- | --- | + | Basic audio-only join channel | Windows | High - missing foundational case | + """ + ) + ) + handle.close() + self.addCleanup(lambda: Path(handle.name).unlink(missing_ok=True)) + return Path(handle.name) + + def test_prepares_one_requirement_with_four_platform_delivery_units(self): + package = prepare_case_execution( + self.write_matrix(), + feature="Join channel audio", + target_sdk_version=self.TARGET_SDK_VERSION, + ) + + manifest = package["acceptance_manifest_seed"] + self.assertEqual(manifest["version"], 4) + self.assertEqual(sorted(manifest["platforms"]), PLATFORMS) + self.assertIn("contract", manifest) + self.assertNotIn("roles", manifest) + self.assertEqual(sorted(package["role_contracts"]), ["contract", "implementation", "verification"]) + self.assertEqual(sorted(manifest["contract"]["output"]["platform_targets"]), PLATFORMS) + self.assertEqual(manifest["requirement"]["target_sdk_version"], self.TARGET_SDK_VERSION) + self.assertTrue(manifest["release"]["required"]) + self.assertEqual(manifest["release"]["target_sdk_version"], self.TARGET_SDK_VERSION) + self.assertNotIn("qa_acceptance", manifest["release"]) + self.assertNotIn("publication_channel", manifest["requirement"]) + self.assertNotIn("publication", manifest["release"]) + for platform in PLATFORMS: + unit = manifest["platforms"][platform] + self.assertEqual(sorted(unit), ["implementation", "verification"]) + self.assertEqual(unit["implementation"]["dispatch"]["mode"], "pending") + self.assertEqual(unit["verification"]["status"], "BLOCKED") + self.assertEqual(validate_manifest(manifest), []) + + def test_platform_defaults_select_official_full_sdk_projects(self): + package = prepare_case_execution( + self.write_matrix(), + feature="Join channel audio", + target_sdk_version=self.TARGET_SDK_VERSION, + ) + targets = package["acceptance_manifest_seed"]["contract"]["output"]["platform_targets"] + + self.assertEqual(targets["android"]["target_project"], "Android/APIExample/") + self.assertEqual(targets["ios"]["target_project"], "iOS/APIExample/") + self.assertEqual(targets["macos"]["target_project"], "macOS/") + self.assertEqual(targets["windows"]["target_project"], "windows/") + self.assertTrue(all(target["required"] for target in targets.values())) + + def test_selects_highest_priority_feature_when_omitted(self): + package = prepare_case_execution( + self.write_matrix(), target_sdk_version=self.TARGET_SDK_VERSION + ) + + self.assertEqual(package["requirement"]["feature"], "Join channel audio") + self.assertEqual(package["requirement"]["key_apis"], ["joinChannel", "setAudioProfile"]) + + def test_package_is_json_serializable(self): + serialized = json.dumps( + prepare_case_execution( + self.write_matrix(), target_sdk_version=self.TARGET_SDK_VERSION + ) + ) + self.assertIn('"version": 4', serialized) + + def test_prepares_new_requirement_not_yet_present_in_matrix(self): + package = prepare_case_execution( + self.write_matrix(), + feature="Spatial audio", + sdk_family="Full RTC", + key_apis=["enableSpatialAudio"], + target_sdk_version=self.TARGET_SDK_VERSION, + ) + + self.assertEqual(package["requirement"]["feature"], "Spatial audio") + self.assertEqual(package["requirement"]["key_apis"], ["enableSpatialAudio"]) + self.assertEqual(sorted(package["acceptance_manifest_seed"]["platforms"]), PLATFORMS) + + def test_requires_target_sdk_version(self): + with self.assertRaisesRegex(ValueError, "target_sdk_version is required"): + prepare_case_execution(self.write_matrix(), feature="Join channel audio") + + def test_collects_live_sdk_version_evidence(self): + for distribution in ["Shengwang", "Agora"]: + with self.subTest(distribution=distribution), tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + profile_path = self.write_repository_profile(root, distribution) + + profile = load_repository_profile(profile_path) + checks = collect_sdk_version_checks( + self.TARGET_SDK_VERSION, + repo_root=root, + sources=profile["sdk_version_sources"], + ) + + self.assertTrue(all(check["result"] == "PASS" for check in checks)) + self.assertTrue( + all( + set(check["actual_versions"].values()) == {self.TARGET_SDK_VERSION} + for check in checks + ) + ) + + archive = root / "windows/install.ps1" + archive.write_text( + f"{distribution}_Native_SDK_for_Windows_v4.6.2_FULL.zip\n", + encoding="utf-8", + ) + checks = collect_sdk_version_checks( + self.TARGET_SDK_VERSION, + repo_root=root, + profile_path=profile_path, + ) + windows = next( + check for check in checks if check["name"] == "sdk-version-windows" + ) + self.assertEqual(windows["result"], "BLOCKED") + self.assertIn("4.6.2", windows["reason"]) + + def write_repository_profile(self, root, distribution): + sources = { + "android": [ + { + "path": "Android/gradle.properties", + "kind": "gradle-property", + "key": "rtc_sdk_version", + } + ], + "ios": [ + { + "path": "iOS/Podfile", + "kind": "cocoapods", + "package": f"{distribution}RtcEngine_iOS", + } + ], + "macos": [ + { + "path": "macOS/Podfile", + "kind": "cocoapods", + "package": f"{distribution}RtcEngine_macOS", + } + ], + "windows": [ + { + "path": "windows/install.ps1", + "kind": "archive-name", + "prefix": f"{distribution}_Native_SDK_for_Windows_v", + "suffix": "_FULL.zip", + } + ], + } + contents = { + "Android/gradle.properties": "rtc_sdk_version = 4.6.4\n", + "iOS/Podfile": f"pod '{distribution}RtcEngine_iOS', '4.6.4'\n", + "macOS/Podfile": f"pod '{distribution}RtcEngine_macOS', '4.6.4'\n", + "windows/install.ps1": ( + f"{distribution}_Native_SDK_for_Windows_v4.6.4_FULL.zip\n" + ), + } + for path_text, content in contents.items(): + path = root / path_text + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + profile_path = root / "repository-profile.json" + profile_path.write_text( + json.dumps({"version": 1, "sdk_version_sources": sources}), + encoding="utf-8", + ) + return profile_path + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ai-engineering/tools/validate_acceptance_manifest.py b/docs/ai-engineering/tools/validate_acceptance_manifest.py new file mode 100644 index 000000000..536c71f9f --- /dev/null +++ b/docs/ai-engineering/tools/validate_acceptance_manifest.py @@ -0,0 +1,1212 @@ +#!/usr/bin/env python3 +"""Validate cross-platform API Examples acceptance manifests.""" + +import hashlib +import json +import re +import shlex +import sys +from pathlib import Path + +from generate_case_backlog import PLATFORM_PROJECTS, parse_matrix_cell + + +REPO_ROOT = Path(__file__).resolve().parents[3] +MANIFEST_VERSION = 4 +PLATFORMS = ["android", "ios", "macos", "windows"] +PLATFORM_ROOTS = { + "android": "Android/", + "ios": "iOS/", + "macos": "macOS/", + "windows": "windows/", +} +FINAL_STATUSES = {"PASS", "PASS WITH RISKS", "BLOCKED"} +RESULT_STATUSES = {"PASS", "FAIL", "BLOCKED", "SKIPPED"} +ROLE_STATUSES = {"PASS", "FAIL", "BLOCKED", "WAIVED"} +MATRIX_STATUSES = {"DONE", "PARTIAL", "MISSING", "N/A", "UNKNOWN"} +REQUIRED_TOP_LEVEL = { + "version", + "final_status", + "requirement", + "contract", + "platforms", + "cross_platform_acceptance", + "release", + "knowledge_updates", +} +REQUIREMENT_FIELDS = {"feature", "sdk_family", "key_apis", "target_sdk_version"} +RELEASE_FIELDS = {"required", "target_sdk_version", "checks", "skipped_checks"} +CONTRACT_OUTPUT_FIELDS = { + "scenario", + "key_apis", + "non_goals", + "reference", + "cross_platform_requirements", + "platform_targets", +} +IMPLEMENTATION_OUTPUT_FIELDS = { + "target_project", + "query_cases", + "upsert_case", + "files_changed", + "matrix_updates", +} +VERIFICATION_OUTPUT_FIELDS = { + "result", + "findings", + "parity_result", + "entry_point", + "ux_notes", + "commands", + "build_result", + "skipped_checks", +} +DISPATCH_MODES = {"codex-exec", "pending"} +COMMAND_KINDS = {"build", "test", "static", "runtime"} +REASONING_EFFORTS = {"none", "minimal", "low", "medium", "high", "xhigh"} +MODEL_PROVENANCE_FIELDS = ["model_profile", "model", "reasoning_effort", "codex_version"] +LEAD_AGENT_IDS = {"lead", "lead-agent", "lead_agent", "main", "main-agent", "main_agent", "coordinator"} +KNOWLEDGE_UPDATE_FIELDS = [ + "source", + "impact_platforms", + "symptom", + "root_cause", + "guardrail", + "verification", + "updated_at", +] +DURABLE_KNOWLEDGE_PATHS = { + "docs/ai-engineering/knowledge-index.md", + "docs/ai-engineering/release-iteration-gate.md", + "docs/ai-engineering/release-known-issues.md", + "docs/ai-engineering/case-maintenance-matrix.md", + "docs/ai-engineering/repository-profile.json", + "docs/ai-engineering/role-routing.json", +} +PLACEHOLDER_RE = re.compile(r"<[^>\n]+>") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +SDK_VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") + + +def is_non_empty(value): + return value is not None and value != "" and value != [] + + +def validate_no_placeholders(value, path, errors): + if isinstance(value, str) and PLACEHOLDER_RE.search(value): + errors.append(f"{path} contains placeholder value: {value}") + elif isinstance(value, dict): + for key, child in value.items(): + validate_no_placeholders(child, f"{path}.{key}", errors) + elif isinstance(value, list): + for index, child in enumerate(value): + validate_no_placeholders(child, f"{path}[{index}]", errors) + + +def validate_manifest_shape(manifest, errors): + for key in sorted(REQUIRED_TOP_LEVEL - set(manifest)): + errors.append(f"missing top-level field: {key}") + for key in sorted(set(manifest) - REQUIRED_TOP_LEVEL): + errors.append(f"unsupported top-level field: {key}") + if manifest.get("version") != MANIFEST_VERSION: + errors.append(f"version must be {MANIFEST_VERSION}") + if manifest.get("final_status") not in FINAL_STATUSES: + errors.append(f"final_status must be one of {sorted(FINAL_STATUSES)}") + requirement = manifest.get("requirement") + if not isinstance(requirement, dict): + errors.append("requirement must be an object") + else: + for field in sorted(REQUIREMENT_FIELDS): + if not is_non_empty(requirement.get(field)): + errors.append(f"requirement.{field} is required") + for field in sorted(set(requirement) - REQUIREMENT_FIELDS): + errors.append(f"unsupported requirement field: {field}") + target_sdk_version = requirement.get("target_sdk_version") + if is_non_empty(target_sdk_version) and not SDK_VERSION_RE.fullmatch( + str(target_sdk_version) + ): + errors.append("requirement.target_sdk_version must use x.y.z format") + + +def validate_manifest(manifest): + if not isinstance(manifest, dict): + return ["manifest must be a JSON object"] + errors = [] + seen_agent_ids = {} + validate_no_placeholders(manifest, "manifest", errors) + validate_manifest_shape(manifest, errors) + targets = validate_contract(manifest, errors, seen_agent_ids) + validate_platforms(manifest, targets, errors, seen_agent_ids) + validate_unique_run_ids(manifest, errors) + validate_cross_platform_acceptance(manifest, errors) + validate_release(manifest, errors) + validate_knowledge_updates(manifest, errors) + return errors + + +def validate_contract(manifest, errors, seen_agent_ids): + contract = manifest.get("contract") + output = validate_artifact( + contract, + "contract", + CONTRACT_OUTPUT_FIELDS, + manifest.get("final_status"), + True, + errors, + seen_agent_ids, + ) + if not output: + return {} + for field in ["scenario", "key_apis"]: + if not is_non_empty(output.get(field)): + errors.append(f"contract.output.{field} is required") + requirement_apis = manifest.get("requirement", {}).get("key_apis", []) + contract_apis = output.get("key_apis", []) + if isinstance(requirement_apis, list) and isinstance(contract_apis, list): + missing_apis = [api for api in requirement_apis if api not in contract_apis] + if missing_apis: + errors.append( + "contract.output.key_apis must include every requirement.key_apis value: " + + ", ".join(str(api) for api in missing_apis) + ) + if contract.get("status") == "PASS" and not is_non_empty( + output.get("cross_platform_requirements") + ): + errors.append("contract.output.cross_platform_requirements is required when Contract passes") + reference = output.get("reference") + if not isinstance(reference, dict): + errors.append("contract.output.reference must be an object") + else: + required = bool(reference.get("required")) + source_case = reference.get("source_case") + if required: + if not is_non_empty(source_case): + errors.append("contract.output.reference.source_case is required") + elif "://" not in source_case: + validate_paths([source_case], "contract.output.reference.source_case", errors) + result = reference.get("contract_result") + validate_result(result, "contract.output.reference.contract_result", manifest.get("final_status"), errors) + if required and contract.get("status") == "PASS" and result != "PASS": + errors.append("reference-required work needs contract.output.reference.contract_result=PASS") + + targets = output.get("platform_targets") + if not isinstance(targets, dict): + errors.append("contract.output.platform_targets must be an object") + return {} + for platform in PLATFORMS: + if platform not in targets: + errors.append(f"contract.output.platform_targets missing platform: {platform}") + continue + target = targets[platform] + if not isinstance(target, dict): + errors.append(f"contract.output.platform_targets.{platform} must be an object") + continue + for field in [ + "required", + "target_project", + "key_constraints", + "files_allowed", + "waiver_reason", + ]: + if field not in target: + errors.append(f"contract.output.platform_targets.{platform}.{field} is required") + if target.get("required") is False and not is_non_empty(target.get("waiver_reason")): + errors.append( + f"contract.output.platform_targets.{platform}.waiver_reason is required when the platform is not required" + ) + if contract.get("status") == "PASS" and target.get("required") is True: + if not is_non_empty(target.get("key_constraints")): + errors.append( + f"contract.output.platform_targets.{platform}.key_constraints is required when Contract passes" + ) + if not is_non_empty(target.get("files_allowed")): + errors.append( + f"contract.output.platform_targets.{platform}.files_allowed is required when Contract passes" + ) + target_project = target.get("target_project") + if is_non_empty(target_project): + validate_paths( + [target_project], + f"contract.output.platform_targets.{platform}.target_project", + errors, + ) + if not path_is_within(target_project, PLATFORM_ROOTS[platform]): + errors.append( + f"contract.output.platform_targets.{platform}.target_project must be inside {PLATFORM_ROOTS[platform]}" + ) + validate_paths( + target.get("files_allowed", []), + f"contract.output.platform_targets.{platform}.files_allowed", + errors, + ) + allowed_paths = target.get("files_allowed", []) + for allowed_path in allowed_paths if isinstance(allowed_paths, list) else []: + if not path_is_within(allowed_path, PLATFORM_ROOTS[platform]): + errors.append( + f"contract.output.platform_targets.{platform}.files_allowed must stay inside {PLATFORM_ROOTS[platform]}: {allowed_path}" + ) + for platform in sorted(set(targets) - set(PLATFORMS)): + errors.append(f"contract.output.platform_targets has unsupported platform: {platform}") + return targets + + +def validate_platforms(manifest, targets, errors, seen_agent_ids): + platforms = manifest.get("platforms") + if not isinstance(platforms, dict): + errors.append("platforms must be an object") + return + for platform in PLATFORMS: + if platform not in platforms: + errors.append(f"platforms missing required platform: {platform}") + continue + unit = platforms[platform] + if not isinstance(unit, dict): + errors.append(f"platforms.{platform} must be an object") + continue + target = targets.get(platform, {}) if isinstance(targets, dict) else {} + required = bool(target.get("required", True)) + implementation = validate_artifact( + unit.get("implementation"), + f"platforms.{platform}.implementation", + IMPLEMENTATION_OUTPUT_FIELDS, + manifest.get("final_status"), + required, + errors, + seen_agent_ids, + ) + verification = validate_artifact( + unit.get("verification"), + f"platforms.{platform}.verification", + VERIFICATION_OUTPUT_FIELDS, + manifest.get("final_status"), + required, + errors, + seen_agent_ids, + ) + validate_platform_implementation(platform, target, implementation, verification, manifest, errors) + verification_dispatch = ( + unit.get("verification", {}).get("dispatch", {}) + if isinstance(unit.get("verification"), dict) + else {} + ) + validate_platform_verification( + platform, required, verification, verification_dispatch, manifest, errors + ) + for role in ["implementation", "verification"]: + artifact = unit.get(role, {}) + dispatch = artifact.get("dispatch", {}) if isinstance(artifact, dict) else {} + working_directory = dispatch.get("working_directory") + expected_working_directory = target.get("target_project") + if ( + is_non_empty(working_directory) + and expected_working_directory + and normalize_manifest_path(working_directory).rstrip("/") + != normalize_manifest_path(expected_working_directory).rstrip("/") + ): + errors.append( + f"platforms.{platform}.{role}.dispatch.working_directory must match Contract target {expected_working_directory}" + ) + if not required: + for role in ["implementation", "verification"]: + artifact = unit.get(role, {}) + if artifact.get("status") != "WAIVED": + errors.append(f"platforms.{platform}.{role}.status must be WAIVED when platform is not required") + for platform in sorted(set(platforms) - set(PLATFORMS)): + errors.append(f"platforms has unsupported platform: {platform}") + + +def validate_artifact(artifact, path, required_output, final_status, required, errors, seen_agent_ids): + if not isinstance(artifact, dict): + errors.append(f"{path} must be an object") + return {} + status = artifact.get("status") + if status not in ROLE_STATUSES: + errors.append(f"{path}.status must be one of {sorted(ROLE_STATUSES)}") + elif required and status == "WAIVED": + errors.append( + f"{path}.status=WAIVED is only valid when the platform is not required" + ) + elif required and status in {"FAIL", "BLOCKED"} and final_status != "BLOCKED": + errors.append(f"{path}.status={status} requires final_status=BLOCKED") + if status == "WAIVED" and not is_non_empty(artifact.get("waiver_reason")): + errors.append(f"{path}.waiver_reason is required when status is WAIVED") + for field in ["agent_id", "evidence", "summary"]: + if not is_non_empty(artifact.get(field)): + errors.append(f"{path}.{field} is required") + agent_id = artifact.get("agent_id") + if is_non_empty(agent_id): + normalized = str(agent_id).lower() + if normalized in LEAD_AGENT_IDS: + errors.append(f"{path}.agent_id cannot be the Lead Agent") + elif normalized in seen_agent_ids: + errors.append(f"{path}.agent_id duplicates {seen_agent_ids[normalized]}.agent_id") + else: + seen_agent_ids[normalized] = path + validate_dispatch(path, artifact.get("dispatch"), status, final_status, required, errors) + if path.endswith(".implementation") and required and final_status != "BLOCKED": + dispatch = artifact.get("dispatch", {}) + if not is_non_empty(dispatch.get("repository_delta")): + errors.append(f"{path}.dispatch.repository_delta is required") + else: + validate_run_relative_path( + dispatch["repository_delta"], f"{path}.dispatch.repository_delta", errors + ) + value = dispatch.get("repository_delta_sha256") + if not isinstance(value, str) or not SHA256_RE.fullmatch(value): + errors.append(f"{path}.dispatch.repository_delta_sha256 must be a lowercase sha256 hex digest") + output = artifact.get("output") + if not isinstance(output, dict): + errors.append(f"{path}.output must be an object") + return {} + for field in sorted(required_output - set(output)): + errors.append(f"{path}.output.{field} is required") + return output + + +def validate_dispatch(path, dispatch, status, final_status, required, errors): + field_name = f"{path}.dispatch" + if not isinstance(dispatch, dict): + errors.append(f"{field_name} is required") + return + mode = dispatch.get("mode") + if mode not in DISPATCH_MODES: + errors.append(f"{field_name}.mode must be one of {sorted(DISPATCH_MODES)}") + for field in ["prompt", "artifact"]: + if not is_non_empty(dispatch.get(field)): + errors.append(f"{field_name}.{field} is required") + else: + validate_run_relative_path(dispatch[field], f"{field_name}.{field}", errors) + if is_non_empty(dispatch.get("input_snapshot")): + validate_run_relative_path(dispatch["input_snapshot"], f"{field_name}.input_snapshot", errors) + if required and mode == "pending" and final_status != "BLOCKED" and status != "WAIVED": + errors.append(f"{field_name}.mode=pending requires final_status=BLOCKED") + if final_status == "BLOCKED" or status == "WAIVED" or not required: + return + if mode != "codex-exec": + errors.append(f"{field_name}.mode={mode} cannot pass final acceptance") + for field in [ + "run_id", + "evidence", + "input_snapshot", + "working_directory", + "command_log", + "host_platform", + *MODEL_PROVENANCE_FIELDS, + ]: + if not is_non_empty(dispatch.get(field)): + errors.append(f"{field_name}.{field} is required when final_status is not BLOCKED") + if is_non_empty(dispatch.get("working_directory")): + validate_paths( + [dispatch["working_directory"]], f"{field_name}.working_directory", errors + ) + if is_non_empty(dispatch.get("command_log")): + validate_run_relative_path( + dispatch["command_log"], f"{field_name}.command_log", errors + ) + effort = dispatch.get("reasoning_effort") + if is_non_empty(effort) and effort not in REASONING_EFFORTS: + errors.append(f"{field_name}.reasoning_effort must be one of {sorted(REASONING_EFFORTS)}") + for field in [ + "prompt_sha256", + "artifact_sha256", + "input_snapshot_sha256", + "command_log_sha256", + ]: + value = dispatch.get(field) + if not is_non_empty(value): + errors.append(f"{field_name}.{field} is required when final_status is not BLOCKED") + elif not isinstance(value, str) or not SHA256_RE.fullmatch(value): + errors.append(f"{field_name}.{field} must be a lowercase sha256 hex digest") + + +def validate_platform_implementation(platform, target, output, verification, manifest, errors): + if not output: + return + expected_target = target.get("target_project") if isinstance(target, dict) else None + if expected_target and output.get("target_project") != expected_target: + errors.append( + f"platforms.{platform}.implementation.output.target_project must match contract target {expected_target}" + ) + for field in ["query_cases", "upsert_case"]: + if not is_non_empty(output.get(field)): + errors.append(f"platforms.{platform}.implementation.output.{field} is required") + validate_paths( + output.get("files_changed", []), + f"platforms.{platform}.implementation.output.files_changed", + errors, + require_exists=False, + ) + allowed = target.get("files_allowed", []) if isinstance(target, dict) else [] + for changed_path in output.get("files_changed", []): + if not any(path_is_within(changed_path, allowed_path) for allowed_path in allowed): + errors.append( + f"platforms.{platform}.implementation.output.files_changed is outside Contract files_allowed: {changed_path}" + ) + updates = output.get("matrix_updates", []) + if not isinstance(updates, list): + errors.append(f"platforms.{platform}.implementation.output.matrix_updates must be an array") + return + for index, update in enumerate(updates): + path = f"platforms.{platform}.implementation.output.matrix_updates[{index}]" + if not isinstance(update, dict): + errors.append(f"{path} must be an object") + continue + for field in ["feature", "platform_unit", "from", "to", "to_cell", "evidence"]: + if not is_non_empty(update.get(field)): + errors.append(f"{path}.{field} is required") + if update.get("feature") != manifest.get("requirement", {}).get("feature"): + errors.append(f"{path}.feature must match requirement.feature") + target_project = target.get("target_project") if isinstance(target, dict) else "" + expected_units = { + unit + for unit, project in PLATFORM_PROJECTS.items() + if path_is_within(target_project, project) or path_is_within(project, target_project) + } + if update.get("platform_unit") not in expected_units: + errors.append( + f"{path}.platform_unit must match Contract target {target_project}; " + f"expected one of {sorted(expected_units)}" + ) + if update.get("from") not in MATRIX_STATUSES: + errors.append(f"{path}.from must be one of {sorted(MATRIX_STATUSES)}") + if update.get("to") not in MATRIX_STATUSES: + errors.append(f"{path}.to must be one of {sorted(MATRIX_STATUSES)}") + to_cell = update.get("to_cell") + if is_non_empty(to_cell): + cell_status, _ = parse_matrix_cell(to_cell) + if cell_status != update.get("to"): + errors.append(f"{path}.to_cell status must match to={update.get('to')}") + if any(character in str(to_cell) for character in "\n\r|`"): + errors.append(f"{path}.to_cell contains unsupported Markdown characters") + if update.get("to") == "DONE": + if manifest.get("final_status") != "PASS": + errors.append(f"{update.get('platform_unit')} matrix update to DONE requires final_status=PASS") + if verification.get("build_result") != "PASS": + errors.append( + f"{update.get('platform_unit')} matrix update to DONE requires build_result=PASS" + ) + if verification.get("parity_result") != "PASS": + errors.append( + f"{update.get('platform_unit')} matrix update to DONE requires parity_result=PASS" + ) + if verification.get("skipped_checks"): + errors.append(f"{update.get('platform_unit')} matrix update to DONE cannot include skipped checks") + if isinstance(target, dict) and target.get("required") is False: + if output.get("files_changed"): + errors.append(f"platforms.{platform}.implementation.output.files_changed must be empty when platform is not required") + if updates: + errors.append(f"platforms.{platform}.implementation.output.matrix_updates must be empty when platform is not required") + + +def is_platform_build_command(platform, command, working_directory=None): + if ( + platform not in PLATFORM_ROOTS + or not isinstance(command, str) + or not is_non_empty(working_directory) + or not path_is_within(working_directory, PLATFORM_ROOTS[platform]) + ): + return False + try: + outer_tokens = shlex.split(command) + except ValueError: + return False + if not outer_tokens: + return False + executable = outer_tokens[0].replace("\\", "/").rsplit("/", 1)[-1].lower() + if executable in {"sh", "bash", "zsh"}: + command_option = next( + ( + index + for index, token in enumerate(outer_tokens[:-1]) + if token.startswith("-") and "c" in token + ), + None, + ) + if command_option is None: + return False + command = outer_tokens[command_option + 1] + elif executable in {"cmd", "cmd.exe", "powershell", "powershell.exe", "pwsh", "pwsh.exe"}: + command_option = next( + ( + index + for index, token in enumerate(outer_tokens[:-1]) + if token.lower() in {"/c", "-command", "-c"} + ), + None, + ) + if command_option is None: + return False + command = " ".join(outer_tokens[command_option + 1 :]) + + try: + lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|") + lexer.whitespace_split = True + lexer.commenters = "" + tokens = list(lexer) + except ValueError: + return False + segments = [[]] + for token in tokens: + if token in {";", "|", "||", "&"}: + return False + if token == "&&": + if not segments[-1]: + return False + segments.append([]) + else: + segments[-1].append(token) + if not segments[-1]: + return False + if len(segments) != 1: + return False + final = segments[-1] + if command_overrides_path(final): + return False + remaining = command_tokens_without_environment(final) + if not remaining: + return False + raw_executable = remaining[0].replace("\\", "/").lower() + final_executable, arguments = command_executable(final), command_arguments(final) + if platform == "android": + if raw_executable not in {"./gradlew", "gradlew.bat", "./gradlew.bat"}: + return False + normalized_arguments = [argument.lower() for argument in arguments] + blocked_modes = { + "--dry-run", + "-m", + "help", + "tasks", + "properties", + "projects", + "dependencies", + "components", + "model", + "buildenvironment", + } + if blocked_modes.intersection(normalized_arguments) or any( + argument.startswith("--dry-run=") for argument in normalized_arguments + ): + return False + if any( + argument in {"-p", "--project-dir"} or argument.startswith("--project-dir=") + for argument in normalized_arguments + ): + return False + return final_executable in {"gradlew", "gradlew.bat"} and any( + re.search(r"(?:^|:)(?:assemble|build|bundle)[A-Za-z0-9_.-]*$", argument, re.I) + for argument in arguments + ) + if platform in {"ios", "macos"}: + if raw_executable not in {"xcodebuild", "/usr/bin/xcodebuild"}: + return False + normalized_arguments = [argument.lower() for argument in arguments] + if not xcodebuild_has_only_build_arguments(arguments): + return False + for option in ["-project", "-workspace"]: + if option in normalized_arguments: + index = normalized_arguments.index(option) + if index + 1 >= len(arguments) or not is_safe_relative_build_path( + arguments[index + 1] + ): + return False + build_actions = {"build", "archive", "test", "build-for-testing"} + normalized_argument_set = set(normalized_arguments) + if normalized_argument_set & build_actions: + return True + if "clean" in normalized_argument_set: + return False + return bool(normalized_argument_set & {"-project", "-workspace", "-scheme"}) + if final_executable in {"msbuild", "msbuild.exe", "devenv", "devenv.com", "devenv.exe"}: + if "/" in raw_executable: + return False + normalized_arguments = [argument.lower() for argument in arguments] + if any( + argument in {"-version", "/version", "-help", "/help", "/?"} + for argument in normalized_arguments + ): + return False + blocked_prefixes = ( + "/preprocess", + "-preprocess", + "/pp", + "-pp", + "/getproperty", + "-getproperty", + "/getitem", + "-getitem", + "/gettargetresult", + "-gettargetresult", + "/targets", + "-targets", + "/ts", + "-ts", + "/validate", + "-validate", + "/val", + "-val", + ) + if any(argument.startswith(blocked_prefixes) for argument in normalized_arguments): + return False + project_arguments = [ + arguments[index] + for index, argument in enumerate(normalized_arguments) + if argument.endswith((".sln", ".slnx", ".vcxproj", ".proj")) + ] + if not project_arguments or not all( + is_safe_relative_build_path(argument) for argument in project_arguments + ): + return False + if final_executable.startswith("devenv"): + return any( + argument in {"/build", "/rebuild"} for argument in normalized_arguments + ) + targets = [ + argument.split(":", 1)[1] + for argument in normalized_arguments + if argument.startswith(("/t:", "-t:", "/target:", "-target:")) + ] + if targets: + target_names = { + name + for value in targets + for name in re.split(r"[;,]", value) + if name + } + return bool(target_names & {"build", "rebuild"}) + return True + if ( + raw_executable not in {"cmake", "cmake.exe"} + or final_executable not in {"cmake", "cmake.exe"} + or "--build" not in arguments + ): + return False + build_index = arguments.index("--build") + if build_index + 1 >= len(arguments) or not is_safe_relative_build_path( + arguments[build_index + 1] + ): + return False + normalized_arguments = [argument.lower() for argument in arguments] + if "--target" in normalized_arguments: + target_index = normalized_arguments.index("--target") + targets = normalized_arguments[target_index + 1 :] + if targets and all(target in {"clean", "help"} for target in targets): + return False + return True + + +def is_safe_relative_build_path(path_text): + if not isinstance(path_text, str) or not path_text: + return False + normalized = path_text.replace("\\", "/") + path = Path(normalized) + return ( + not path.is_absolute() + and not re.match(r"^[A-Za-z]:/", normalized) + and ".." not in path.parts + ) + + +def xcodebuild_has_only_build_arguments(arguments): + actions = {"build", "archive", "test", "build-for-testing", "clean"} + options_with_values = { + "-project", + "-workspace", + "-scheme", + "-target", + "-configuration", + "-destination", + "-arch", + "-sdk", + "-xcconfig", + "-deriveddatapath", + "-archivepath", + "-resultbundlepath", + "-resultstreampath", + "-packagecachepath", + "-clonedsourcepackagesdirpath", + "-jobs", + "-testplan", + "-only-testing", + "-skip-testing", + "-enablecodecoverage", + "-parallel-testing-enabled", + "-maximum-concurrent-test-device-destinations", + "-maximum-concurrent-test-simulator-destinations", + } + flags = { + "-alltargets", + "-parallelizetargets", + "-quiet", + "-verbose", + "-hideshellscriptenvironment", + "-allowprovisioningupdates", + "-allowprovisioningdeviceregistration", + "-skipunavailableactions", + "-disableautomaticpackageresolution", + "-onlyusepackageversionsfromresolvedfile", + "-skippackageupdates", + "-skippackagepluginvalidation", + "-skipmacrovalidation", + "-showbuildtimingsummary", + } + index = 0 + while index < len(arguments): + argument = arguments[index] + normalized = argument.lower() + if normalized in actions or normalized in flags or ( + "=" in argument and not argument.startswith("-") + ): + index += 1 + continue + if normalized in options_with_values: + if index + 1 >= len(arguments): + return False + index += 2 + continue + return False + return True + + +def command_tokens_without_environment(tokens): + remaining = list(tokens) + if remaining and remaining[0] == "env": + remaining.pop(0) + while remaining and (remaining[0].startswith("-") or "=" in remaining[0]): + remaining.pop(0) + while remaining and "=" in remaining[0] and not remaining[0].startswith(("/", "./")): + remaining.pop(0) + return remaining + + +def command_overrides_path(tokens): + remaining = list(tokens) + if remaining and remaining[0] == "env": + remaining.pop(0) + while remaining and remaining[0].startswith("-"): + remaining.pop(0) + for token in remaining: + if "=" not in token or token.startswith(("/", "./")): + break + name, _, _ = token.partition("=") + if name.upper() == "PATH": + return True + return False + + +def command_executable(tokens): + remaining = command_tokens_without_environment(tokens) + if not remaining: + return "" + return remaining[0].replace("\\", "/").rsplit("/", 1)[-1].lower() + + +def command_arguments(tokens): + remaining = command_tokens_without_environment(tokens) + return remaining[1:] if remaining else [] + + +def validate_platform_verification( + platform, required, output, dispatch, manifest, errors +): + if not output: + return + dispatch = dispatch if isinstance(dispatch, dict) else {} + command_log = dispatch.get("command_log") + host_platform = dispatch.get("host_platform") + for field in ["entry_point", "ux_notes"]: + if required and not is_non_empty(output.get(field)): + errors.append(f"platforms.{platform}.verification.output.{field} is required") + for field in ["result", "parity_result", "build_result"]: + validate_result( + output.get(field), + f"platforms.{platform}.verification.output.{field}", + manifest.get("final_status"), + errors, + allow_skipped=not required, + ) + commands = output.get("commands", []) + if not required: + for field in ["result", "parity_result", "build_result"]: + if output.get(field) != "SKIPPED": + errors.append( + f"platforms.{platform}.verification.output.{field} must be SKIPPED when platform is not required" + ) + if commands: + errors.append( + f"platforms.{platform}.verification.output.commands must be empty when platform is not required" + ) + if required and manifest.get("final_status") != "BLOCKED" and not commands: + errors.append(f"platforms.{platform}.verification.output.commands must not be empty") + for index, command in enumerate(commands): + if not isinstance(command, dict): + errors.append(f"platforms.{platform}.verification.output.commands[{index}] must be an object") + continue + if not command.get("command"): + errors.append(f"platforms.{platform}.verification.output.commands[{index}].command is required") + if command.get("kind") not in COMMAND_KINDS: + errors.append( + f"platforms.{platform}.verification.output.commands[{index}].kind must be one of {sorted(COMMAND_KINDS)}" + ) + elif command.get("kind") == "build" and not is_platform_build_command( + platform, command.get("command"), dispatch.get("working_directory") + ): + errors.append( + f"platforms.{platform}.verification.output.commands[{index}] is not a recognized {platform} build command" + ) + validate_result( + command.get("result"), + f"platforms.{platform}.verification.output.commands[{index}].result", + manifest.get("final_status"), + errors, + ) + if command.get("result") in {"PASS", "FAIL"} and not is_non_empty( + command.get("evidence") + ): + errors.append( + f"platforms.{platform}.verification.output.commands[{index}].evidence is required for executed commands" + ) + elif command.get("result") in {"PASS", "FAIL"} and ( + not command_log + or not str(command.get("evidence")).startswith(f"{command_log}#") + or "exit_code=" not in str(command.get("evidence")) + ): + errors.append( + f"platforms.{platform}.verification.output.commands[{index}].evidence must bind to dispatch.command_log and exit_code" + ) + for index, skipped in enumerate(output.get("skipped_checks", [])): + if not isinstance(skipped, dict) or not skipped.get("reason"): + errors.append( + f"platforms.{platform}.verification.output.skipped_checks[{index}].reason is required" + ) + if required and manifest.get("final_status") != "BLOCKED": + for field in ["result", "parity_result", "build_result"]: + if output.get(field) != "PASS": + errors.append( + f"non-BLOCKED acceptance requires platforms.{platform}.verification.output.{field}=PASS" + ) + if output.get("build_result") == "PASS" and not any( + command.get("kind") == "build" and command.get("result") == "PASS" + for command in commands + if isinstance(command, dict) + ): + errors.append( + f"platforms.{platform}.verification.output.build_result=PASS requires a bound PASS build command" + ) + if platform == "windows" and output.get("build_result") == "PASS" and host_platform != "win32": + errors.append( + "platforms.windows.verification.output.build_result=PASS requires host_platform=win32" + ) + if required and manifest.get("final_status") == "PASS" and output.get("skipped_checks"): + errors.append("final_status=PASS cannot include skipped checks") + + +def validate_cross_platform_acceptance(manifest, errors): + acceptance = manifest.get("cross_platform_acceptance") + if not isinstance(acceptance, dict): + errors.append("cross_platform_acceptance must be an object") + return + result = acceptance.get("result") + validate_result(result, "cross_platform_acceptance.result", manifest.get("final_status"), errors) + if not is_non_empty(acceptance.get("evidence")): + errors.append("cross_platform_acceptance.evidence is required") + if manifest.get("final_status") != "BLOCKED" and result != "PASS": + errors.append("non-BLOCKED acceptance requires cross_platform_acceptance.result=PASS") + + +def validate_result(result, field_name, final_status, errors, allow_skipped=False): + if result not in RESULT_STATUSES: + errors.append(f"{field_name} must be one of {sorted(RESULT_STATUSES)}") + elif result in {"FAIL", "BLOCKED"} and final_status != "BLOCKED": + errors.append(f"{field_name}={result} requires final_status=BLOCKED") + elif result == "SKIPPED" and final_status == "PASS" and not allow_skipped: + errors.append(f"{field_name}=SKIPPED cannot use final_status=PASS") + + +def validate_release(manifest, errors): + release = manifest.get("release") + if not isinstance(release, dict): + errors.append("release must be an object") + return + if release.get("required") is not True: + errors.append("release.required must be true") + for field in sorted(set(release) - RELEASE_FIELDS): + errors.append(f"unsupported release field: {field}") + requirement = manifest.get("requirement", {}) + target_sdk_version = requirement.get("target_sdk_version") + if release.get("target_sdk_version") != target_sdk_version: + errors.append("release.target_sdk_version must match requirement.target_sdk_version") + checks = release.get("checks", []) + if not checks: + errors.append("release.checks must include SDK version checks") + check_names = { + check.get("name") for check in checks if isinstance(check, dict) and check.get("name") + } + expected_sdk_checks = {f"sdk-version-{platform}" for platform in PLATFORMS} + for name in sorted(expected_sdk_checks - check_names): + errors.append(f"release.checks missing required check: {name}") + for index, check in enumerate(checks): + if not isinstance(check, dict): + errors.append(f"release.checks[{index}] must be an object") + continue + if not check.get("name"): + errors.append(f"release.checks[{index}].name is required") + result = check.get("result") + validate_result(result, f"release.checks[{index}].result", manifest.get("final_status"), errors) + if result == "PASS" and not check.get("evidence"): + errors.append(f"release.checks[{index}].evidence is required when result is PASS") + if result == "SKIPPED" and not check.get("reason"): + errors.append(f"release.checks[{index}].reason is required when result is SKIPPED") + if result in {"FAIL", "BLOCKED"} and not check.get("reason"): + errors.append(f"release.checks[{index}].reason is required when result is {result}") + if check.get("name") in expected_sdk_checks: + if check.get("expected_version") != target_sdk_version: + errors.append( + f"{check.get('name')} expected_version must match {target_sdk_version}" + ) + actual_versions = check.get("actual_versions") + if not isinstance(actual_versions, dict) or not actual_versions: + errors.append(f"{check.get('name')} actual_versions must be a non-empty object") + elif result == "PASS" and any( + version != target_sdk_version for version in actual_versions.values() + ): + errors.append( + f"{check.get('name')} actual versions must all match {target_sdk_version}" + ) + if manifest.get("final_status") != "BLOCKED" and result != "PASS": + errors.append( + f"non-BLOCKED acceptance requires {check.get('name')}=PASS" + ) + for index, skipped in enumerate(release.get("skipped_checks", [])): + if not isinstance(skipped, dict) or not skipped.get("reason"): + errors.append(f"release.skipped_checks[{index}].reason is required") + if manifest.get("final_status") != "BLOCKED" and release.get("skipped_checks"): + errors.append("non-BLOCKED acceptance cannot include skipped release checks") + +def validate_knowledge_updates(manifest, errors): + updates = manifest.get("knowledge_updates", []) + for index, update in enumerate(updates): + if not isinstance(update, dict): + errors.append(f"knowledge_updates[{index}] must be an object") + continue + for field in KNOWLEDGE_UPDATE_FIELDS: + if not is_non_empty(update.get(field)): + errors.append(f"knowledge_updates[{index}].{field} is required") + changed_files = [] + platforms = manifest.get("platforms", {}) + if isinstance(platforms, dict): + for platform in PLATFORMS: + implementation = platforms.get(platform, {}).get("implementation", {}) + output = implementation.get("output", {}) if isinstance(implementation, dict) else {} + changed_files.extend(output.get("files_changed", []) if isinstance(output, dict) else []) + if updates and not any(is_durable_knowledge_path(path) for path in changed_files): + errors.append("knowledge_updates require at least one durable knowledge doc or skill in files_changed") + + +def validate_run_relative_path(path_text, field_name, errors): + if not isinstance(path_text, str): + errors.append(f"{field_name} must be a string") + return + path = Path(path_text) + if path.is_absolute() or ".." in path.parts: + errors.append(f"{field_name} must be relative to the execution workspace") + + +def validate_unique_run_ids(manifest, errors): + seen = {} + for artifact_path, artifact in iter_manifest_artifacts(manifest): + dispatch = artifact.get("dispatch") + if not isinstance(dispatch, dict) or dispatch.get("mode") != "codex-exec": + continue + run_id = dispatch.get("run_id") + if not is_non_empty(run_id): + continue + normalized = str(run_id).lower() + if normalized in seen: + errors.append( + f"{artifact_path}.dispatch.run_id duplicates {seen[normalized]}.dispatch.run_id" + ) + else: + seen[normalized] = artifact_path + + +def resolve_repo_path(path_text): + path = Path(path_text) + resolved = path.resolve() if path.is_absolute() else (REPO_ROOT / path).resolve() + try: + resolved.relative_to(REPO_ROOT.resolve()) + except ValueError: + return None + return resolved + + +def validate_paths(values, field_name, errors, require_exists=True): + if not isinstance(values, list): + errors.append(f"{field_name} must be an array") + return + for path_text in values: + if not isinstance(path_text, str) or not path_text: + errors.append(f"{field_name} contains an empty or non-string path") + continue + if "://" in path_text: + continue + resolved = resolve_repo_path(path_text) + if resolved is None: + errors.append(f"{field_name} path must be inside the repository: {path_text}") + elif require_exists and not resolved.exists(): + errors.append(f"{field_name} path does not exist: {path_text}") + + +def is_durable_knowledge_path(path_text): + if not isinstance(path_text, str): + return False + normalized = normalize_manifest_path(path_text) + return ( + normalized in DURABLE_KNOWLEDGE_PATHS + or normalized.endswith("/ARCHITECTURE.md") + or normalized.startswith(".agent/skills/") + or "/.agent/skills/" in normalized + or normalized.startswith(".agents/skills/") + or "/.agents/skills/" in normalized + ) + + +def normalize_manifest_path(path_text): + normalized = str(path_text).replace("\\", "/") + path = Path(normalized) + if path.is_absolute(): + try: + return path.resolve().relative_to(REPO_ROOT).as_posix() + except ValueError: + return path.as_posix() + try: + return (REPO_ROOT / path).resolve().relative_to(REPO_ROOT.resolve()).as_posix() + except ValueError: + return path.as_posix() + + +def path_is_within(path_text, allowed_text): + path = normalize_manifest_path(path_text).rstrip("/") + allowed = normalize_manifest_path(allowed_text).rstrip("/") + return path == allowed or path.startswith(f"{allowed}/") + + +def iter_manifest_artifacts(manifest): + contract = manifest.get("contract") + if isinstance(contract, dict): + yield "contract", contract + platforms = manifest.get("platforms") + if not isinstance(platforms, dict): + return + for platform in PLATFORMS: + unit = platforms.get(platform) + if not isinstance(unit, dict): + continue + for role in ["implementation", "verification"]: + artifact = unit.get(role) + if isinstance(artifact, dict): + yield f"platforms.{platform}.{role}", artifact + + +def sha256_file(path): + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def resolve_evidence_path(root, path_text): + if not isinstance(path_text, str): + return None + path = Path(path_text) + if path.is_absolute() or ".." in path.parts: + return None + resolved = (root / path).resolve() + try: + resolved.relative_to(root.resolve()) + except ValueError: + return None + return resolved + + +def validate_evidence_files(manifest, evidence_root): + errors = [] + root = Path(evidence_root).resolve() + fields = [ + ("prompt", "prompt_sha256"), + ("artifact", "artifact_sha256"), + ("input_snapshot", "input_snapshot_sha256"), + ("command_log", "command_log_sha256"), + ("repository_delta", "repository_delta_sha256"), + ] + for artifact_path, artifact in iter_manifest_artifacts(manifest): + dispatch = artifact.get("dispatch") + if not isinstance(dispatch, dict) or dispatch.get("mode") != "codex-exec": + continue + for path_field, hash_field in fields: + path_text = dispatch.get(path_field) + expected = dispatch.get(hash_field) + if not is_non_empty(path_text) or not is_non_empty(expected): + continue + path = resolve_evidence_path(root, path_text) + field_name = f"{artifact_path}.dispatch.{path_field}" + if path is None: + errors.append(f"{field_name} must resolve inside the execution workspace") + elif not path.is_file(): + errors.append(f"{field_name} does not exist: {path_text}") + elif sha256_file(path) != expected: + errors.append( + f"{artifact_path}.dispatch.{hash_field} does not match {path_text}" + ) + elif path_field == "repository_delta": + try: + delta = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + errors.append(f"{field_name} is not valid JSON: {exc}") + continue + if not isinstance(delta, dict): + errors.append(f"{field_name} must contain a JSON object") + continue + delta_files = { + normalize_manifest_path(value) + for value in delta.get("changed_files", []) + if isinstance(value, str) + } + output = artifact.get("output", {}) + artifact_files = { + normalize_manifest_path(value) + for value in output.get("files_changed", []) + if isinstance(value, str) + } + if delta_files != artifact_files: + errors.append( + f"{artifact_path}.dispatch.repository_delta changed_files do not match output.files_changed" + ) + return errors + + +def main(argv): + if len(argv) != 2: + print("usage: validate_acceptance_manifest.py ", file=sys.stderr) + return 2 + manifest_path = Path(argv[1]) + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except OSError as exc: + print(f"failed to read manifest: {exc}", file=sys.stderr) + return 2 + except json.JSONDecodeError as exc: + print(f"invalid JSON: {exc}", file=sys.stderr) + return 2 + errors = validate_manifest(manifest) + errors.extend(validate_evidence_files(manifest, manifest_path.parent)) + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print("Acceptance manifest valid") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/docs/ai-engineering/tools/validate_acceptance_manifest_test.py b/docs/ai-engineering/tools/validate_acceptance_manifest_test.py new file mode 100644 index 000000000..464788e72 --- /dev/null +++ b/docs/ai-engineering/tools/validate_acceptance_manifest_test.py @@ -0,0 +1,700 @@ +import copy +import hashlib +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +TOOLS_DIR = Path(__file__).resolve().parent +if str(TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(TOOLS_DIR)) + +from validate_acceptance_manifest import ( + is_durable_knowledge_path, + validate_evidence_files, + validate_manifest, +) + + +PLATFORMS = ["android", "ios", "macos", "windows"] +TARGETS = { + "android": "Android/APIExample/", + "ios": "iOS/APIExample/", + "macos": "macOS/", + "windows": "windows/", +} +SNAPSHOT_SHA256 = "c" * 64 +BUILD_COMMANDS = { + "android": "./gradlew assembleDebug", + "ios": "xcodebuild -scheme APIExample build", + "macos": "xcodebuild -scheme APIExample build", + "windows": "msbuild APIExample.sln /t:Build", +} + + +def dispatch_for(name, profile): + platform = name.split("-")[0] + dispatch = { + "mode": "codex-exec", + "run_id": f"run-{name}", + "prompt": f"role-prompts/{name}.md", + "prompt_sha256": "a" * 64, + "artifact": f"role-artifacts/{name}.json", + "artifact_sha256": "b" * 64, + "evidence": f"Independent {name} run.", + "model_profile": profile, + "model": "test-model", + "reasoning_effort": "high" if profile in {"deep", "review"} else "medium", + "codex_version": "codex-cli test", + "input_snapshot": f"input-snapshots/{name}.json", + "input_snapshot_sha256": SNAPSHOT_SHA256, + "command_log": f"dispatch-logs/{name}.jsonl", + "command_log_sha256": "d" * 64, + "working_directory": TARGETS.get(name.split("-")[0], "."), + "host_platform": "win32" if platform == "windows" else "darwin", + } + if name.endswith("-implementation"): + dispatch["repository_delta"] = f"repository-deltas/{name}.json" + dispatch["repository_delta_sha256"] = "e" * 64 + return dispatch + + +def artifact(name, profile, output): + return { + "agent_id": f"{name}-agent-1", + "dispatch": dispatch_for(name, profile), + "status": "PASS", + "evidence": f"{name} evidence", + "summary": f"{name} completed", + "output": output, + } + + +def base_manifest(): + platform_targets = { + platform: { + "required": True, + "target_project": target, + "key_constraints": [f"Keep changes in {target}"], + "files_allowed": [target], + "waiver_reason": "", + } + for platform, target in TARGETS.items() + } + platforms = {} + for platform, target in TARGETS.items(): + platforms[platform] = { + "implementation": artifact( + f"{platform}-implementation", + "deep", + { + "target_project": target, + "query_cases": "No duplicate case found.", + "upsert_case": "Platform case updated.", + "files_changed": [f"{target}AGENTS.md"], + "matrix_updates": [], + }, + ), + "verification": artifact( + f"{platform}-verification", + "review", + { + "result": "PASS", + "findings": ["No blocking findings."], + "parity_result": "PASS", + "entry_point": "Basic > Join channel audio", + "ux_notes": "Matches adjacent examples.", + "commands": [ + { + "kind": "build", + "command": BUILD_COMMANDS[platform], + "result": "PASS", + "evidence": ( + f"dispatch-logs/{platform}-verification.jsonl#build; exit_code=0" + ), + } + ], + "build_result": "PASS", + "skipped_checks": [], + }, + ), + } + return { + "version": 4, + "final_status": "PASS", + "requirement": { + "feature": "Join channel audio", + "sdk_family": "Full RTC", + "key_apis": ["joinChannel", "setAudioProfile"], + "target_sdk_version": "4.6.4", + }, + "contract": artifact( + "contract", + "standard", + { + "scenario": "Keep the official audio-only join sample aligned on every platform.", + "key_apis": ["joinChannel", "setAudioProfile"], + "non_goals": ["Release packaging"], + "reference": { + "required": True, + "source_case": "Android/APIExample/app/src/main/java/io/agora/api/example/examples/basic/JoinChannelAudio.java", + "contract_result": "PASS", + }, + "cross_platform_requirements": ["Equivalent join, leave, mute, and status behavior"], + "platform_targets": platform_targets, + }, + ), + "platforms": platforms, + "cross_platform_acceptance": { + "result": "PASS", + "evidence": "All four required platform verification artifacts passed.", + "differences": [], + }, + "release": { + "required": True, + "target_sdk_version": "4.6.4", + "checks": [ + { + "name": f"sdk-version-{platform}", + "result": "PASS", + "expected_version": "4.6.4", + "actual_versions": {f"{platform}/version-source": "4.6.4"}, + "evidence": f"{platform} SDK sources match 4.6.4.", + "reason": "", + } + for platform in PLATFORMS + ], + "skipped_checks": [], + }, + "knowledge_updates": [], + } + + +class AcceptanceManifestValidatorTest(unittest.TestCase): + def assert_error_contains(self, manifest, expected): + errors = validate_manifest(manifest) + self.assertTrue(any(expected in error for error in errors), errors) + + def test_accepts_multi_platform_v4_manifest(self): + self.assertEqual(validate_manifest(base_manifest()), []) + + def test_rejects_legacy_single_role_tree(self): + manifest = base_manifest() + manifest["roles"] = {} + + self.assert_error_contains(manifest, "unsupported top-level field: roles") + + def test_requires_all_official_platform_units(self): + manifest = base_manifest() + del manifest["platforms"]["windows"] + + self.assert_error_contains(manifest, "platforms missing required platform: windows") + + def test_required_platform_blocker_forces_global_blocked(self): + manifest = base_manifest() + manifest["platforms"]["windows"]["verification"]["status"] = "BLOCKED" + + self.assert_error_contains( + manifest, "platforms.windows.verification.status=BLOCKED requires final_status=BLOCKED" + ) + + def test_required_role_cannot_be_waived(self): + manifest = base_manifest() + artifact = manifest["platforms"]["android"]["verification"] + artifact["status"] = "WAIVED" + artifact["waiver_reason"] = "No Android worker." + + self.assert_error_contains( + manifest, + "platforms.android.verification.status=WAIVED is only valid when the platform is not required", + ) + + def test_blocked_manifest_accepts_pending_windows_verification(self): + manifest = base_manifest() + manifest["final_status"] = "BLOCKED" + manifest["cross_platform_acceptance"] = { + "result": "BLOCKED", + "evidence": "Windows-host verification is pending.", + "differences": [], + } + verification = manifest["platforms"]["windows"]["verification"] + verification.update( + { + "agent_id": "windows-verification-agent-pending", + "dispatch": { + "mode": "pending", + "prompt": "role-prompts/windows-verification.md", + "artifact": "role-artifacts/windows-verification.json", + }, + "status": "BLOCKED", + "evidence": "Windows target build requires a Windows host.", + "summary": "Windows verification pending.", + "output": { + "result": "BLOCKED", + "findings": [], + "parity_result": "BLOCKED", + "entry_point": "Pending Windows verification.", + "ux_notes": "Pending.", + "commands": [], + "build_result": "BLOCKED", + "skipped_checks": [ + {"name": "Windows MSBuild", "reason": "Current host is macOS."} + ], + }, + } + ) + + self.assertEqual(validate_manifest(manifest), []) + + def test_non_required_platform_accepts_explicit_waiver(self): + manifest = base_manifest() + target = manifest["contract"]["output"]["platform_targets"]["windows"] + target["required"] = False + target["waiver_reason"] = "The requested SDK family is not available on Windows." + for role in ["implementation", "verification"]: + artifact = manifest["platforms"]["windows"][role] + artifact["status"] = "WAIVED" + artifact["waiver_reason"] = target["waiver_reason"] + artifact["dispatch"] = { + "mode": "pending", + "prompt": f"role-prompts/windows-{role}.md", + "artifact": f"role-artifacts/windows-{role}.json", + } + implementation = manifest["platforms"]["windows"]["implementation"]["output"] + implementation["files_changed"] = [] + implementation["matrix_updates"] = [] + verification = manifest["platforms"]["windows"]["verification"]["output"] + verification["result"] = "SKIPPED" + verification["parity_result"] = "SKIPPED" + verification["build_result"] = "SKIPPED" + verification["commands"] = [] + verification["skipped_checks"] = [] + + self.assertEqual(validate_manifest(manifest), []) + + def test_cross_platform_acceptance_must_pass_for_non_blocked_result(self): + manifest = base_manifest() + manifest["cross_platform_acceptance"]["result"] = "BLOCKED" + + self.assert_error_contains(manifest, "non-BLOCKED acceptance requires cross_platform_acceptance.result=PASS") + + def test_contract_and_implementation_target_must_match(self): + manifest = base_manifest() + manifest["platforms"]["ios"]["implementation"]["output"]["target_project"] = "iOS/APIExample-OC/" + + self.assert_error_contains( + manifest, "platforms.ios.implementation.output.target_project must match contract target" + ) + + def test_contract_must_cover_requirement_key_apis(self): + manifest = base_manifest() + manifest["contract"]["output"]["key_apis"] = ["joinChannel"] + + self.assert_error_contains( + manifest, + "contract.output.key_apis must include every requirement.key_apis value: setAudioProfile", + ) + + def test_passing_contract_requires_cross_platform_constraints(self): + manifest = base_manifest() + manifest["contract"]["output"]["cross_platform_requirements"] = [] + + self.assert_error_contains( + manifest, + "contract.output.cross_platform_requirements is required when Contract passes", + ) + + def test_implementation_requires_query_and_upsert_results(self): + manifest = base_manifest() + manifest["platforms"]["android"]["implementation"]["output"]["query_cases"] = "" + + self.assert_error_contains( + manifest, + "platforms.android.implementation.output.query_cases is required", + ) + + def test_required_verification_requires_entry_point_and_ux_notes(self): + manifest = base_manifest() + manifest["platforms"]["ios"]["verification"]["output"]["entry_point"] = "" + + self.assert_error_contains( + manifest, + "platforms.ios.verification.output.entry_point is required", + ) + + def test_duplicate_agent_ids_across_platforms_are_rejected(self): + manifest = base_manifest() + manifest["platforms"]["windows"]["verification"]["agent_id"] = manifest["platforms"]["ios"][ + "verification" + ]["agent_id"] + + self.assert_error_contains(manifest, "duplicates platforms.ios.verification.agent_id") + + def test_duplicate_codex_run_ids_across_roles_are_rejected(self): + manifest = base_manifest() + implementation_run_id = manifest["platforms"]["ios"]["implementation"]["dispatch"][ + "run_id" + ] + manifest["platforms"]["ios"]["verification"]["dispatch"][ + "run_id" + ] = implementation_run_id + + self.assert_error_contains( + manifest, + "platforms.ios.verification.dispatch.run_id duplicates platforms.ios.implementation.dispatch.run_id", + ) + + def test_pass_rejects_skipped_platform_checks(self): + manifest = base_manifest() + manifest["platforms"]["windows"]["verification"]["output"]["skipped_checks"] = [ + {"name": "runtime", "reason": "No device"} + ] + + self.assert_error_contains(manifest, "final_status=PASS cannot include skipped checks") + + def test_done_matrix_update_requires_platform_build_pass(self): + manifest = base_manifest() + manifest["platforms"]["windows"]["implementation"]["output"]["matrix_updates"] = [ + { + "feature": "Join channel audio", + "platform_unit": "Windows", + "from": "MISSING", + "to": "DONE", + "to_cell": "DONE(Basic/JoinChannelAudio/)", + "evidence": "Implemented.", + } + ] + manifest["platforms"]["windows"]["verification"]["output"]["build_result"] = "SKIPPED" + + self.assert_error_contains(manifest, "Windows matrix update to DONE requires build_result=PASS") + + def test_matrix_update_must_match_current_requirement(self): + manifest = base_manifest() + manifest["platforms"]["windows"]["implementation"]["output"]["matrix_updates"] = [ + { + "feature": "Media metadata", + "platform_unit": "Windows", + "from": "MISSING", + "to": "PARTIAL", + "to_cell": "PARTIAL(pending)", + "evidence": "Unrelated update.", + } + ] + + self.assert_error_contains( + manifest, + "platforms.windows.implementation.output.matrix_updates[0].feature must match requirement.feature", + ) + + def test_matrix_update_must_match_contract_target_project(self): + manifest = base_manifest() + manifest["platforms"]["windows"]["implementation"]["output"]["matrix_updates"] = [ + { + "feature": "Join channel audio", + "platform_unit": "Android full", + "from": "MISSING", + "to": "PARTIAL", + "to_cell": "PARTIAL(wrong platform)", + "evidence": "Wrong platform update.", + } + ] + + self.assert_error_contains( + manifest, + "platforms.windows.implementation.output.matrix_updates[0].platform_unit must match Contract target", + ) + + def test_matrix_update_cell_status_must_match_structured_status(self): + manifest = base_manifest() + manifest["final_status"] = "PASS WITH RISKS" + manifest["platforms"]["android"]["verification"]["output"]["skipped_checks"] = [ + {"name": "device-smoke", "reason": "Device unavailable."} + ] + manifest["platforms"]["android"]["implementation"]["output"]["matrix_updates"] = [ + { + "feature": "Join channel audio", + "platform_unit": "Android full", + "from": "MISSING", + "to": "PARTIAL", + "to_cell": "DONE(fake/path)", + "evidence": "Runtime smoke is pending.", + } + ] + + self.assert_error_contains( + manifest, + "platforms.android.implementation.output.matrix_updates[0].to_cell status must match to=PARTIAL", + ) + + def test_release_cannot_be_disabled(self): + manifest = base_manifest() + manifest["release"]["required"] = False + + self.assert_error_contains(manifest, "release.required must be true") + + def test_rejects_removed_publication_fields(self): + manifest = base_manifest() + manifest["requirement"]["publication_channel"] = "legacy" + manifest["release"]["publication"] = {} + + self.assert_error_contains(manifest, "unsupported requirement field: publication_channel") + self.assert_error_contains(manifest, "unsupported release field: publication") + + def test_release_target_version_must_match_requirement(self): + manifest = base_manifest() + manifest["release"]["target_sdk_version"] = "4.6.3" + + self.assert_error_contains( + manifest, "release.target_sdk_version must match requirement.target_sdk_version" + ) + + def test_target_sdk_version_requires_semver_triplet(self): + manifest = base_manifest() + manifest["requirement"]["target_sdk_version"] = "next" + manifest["release"]["target_sdk_version"] = "next" + + self.assert_error_contains(manifest, "requirement.target_sdk_version must use x.y.z format") + + def test_release_sdk_check_requires_all_sources_to_match_target(self): + manifest = base_manifest() + check = manifest["release"]["checks"][0] + check["actual_versions"]["android/version-source"] = "4.6.3" + + self.assert_error_contains(manifest, "sdk-version-android actual versions must all match 4.6.4") + + def test_rejects_external_ci_and_qa_metadata(self): + manifest = base_manifest() + manifest["release"]["qa_acceptance"] = {} + + self.assert_error_contains(manifest, "unsupported release field: qa_acceptance") + + def test_non_blocked_release_rejects_skipped_checks(self): + manifest = base_manifest() + manifest["final_status"] = "PASS WITH RISKS" + manifest["release"]["skipped_checks"] = [ + {"name": "qa-device-smoke", "reason": "Device unavailable"} + ] + + self.assert_error_contains( + manifest, + "non-BLOCKED acceptance cannot include skipped release checks", + ) + + def test_build_pass_requires_bound_command_evidence(self): + manifest = base_manifest() + del manifest["platforms"]["android"]["verification"]["output"]["commands"][0][ + "evidence" + ] + + self.assert_error_contains( + manifest, + "platforms.android.verification.output.commands[0].evidence is required", + ) + + def test_successful_non_build_command_cannot_satisfy_build_result(self): + manifest = base_manifest() + command = manifest["platforms"]["android"]["verification"]["output"]["commands"][0] + command["command"] = "true" + + self.assert_error_contains( + manifest, + "platforms.android.verification.output.commands[0] is not a recognized android build command", + ) + + def test_build_tool_name_in_echo_cannot_satisfy_build_result(self): + manifest = base_manifest() + command = manifest["platforms"]["android"]["verification"]["output"]["commands"][0] + command["command"] = "echo ./gradlew assembleDebug" + + self.assert_error_contains( + manifest, + "platforms.android.verification.output.commands[0] is not a recognized android build command", + ) + + def test_non_build_tool_modes_cannot_satisfy_build_result(self): + for platform, command_text in [ + ("android", "./gradlew --dry-run assembleDebug"), + ("android", "./gradlew --dry-run=true assembleDebug"), + ("android", "./gradlew help --task assembleDebug"), + ("ios", "xcodebuild -version"), + ("ios", "xcodebuild -scheme APIExample -showTestPlans"), + ("ios", "xcodebuild -scheme APIExample -resolvePackageDependencies"), + ("ios", "xcodebuild -scheme APIExample -checkFirstLaunchStatus"), + ("macos", "xcodebuild -showsdks"), + ("windows", "msbuild -version"), + ("windows", "msbuild APIExample.sln /t:Clean"), + ("windows", "msbuild APIExample.sln /preprocess:out.xml"), + ("windows", "msbuild APIExample.sln /targets"), + ("windows", "msbuild APIExample.sln /validate"), + ("windows", "cmake --build build --target help"), + ("android", "../fake/gradlew assembleDebug"), + ("ios", "/tmp/xcodebuild -scheme APIExample build"), + ("windows", "PATH=./fake msbuild APIExample.sln /t:Build"), + ]: + with self.subTest(platform=platform): + manifest = base_manifest() + command = manifest["platforms"][platform]["verification"]["output"]["commands"][0] + command["command"] = command_text + + self.assert_error_contains( + manifest, + f"platforms.{platform}.verification.output.commands[0] is not a recognized {platform} build command", + ) + + def test_build_command_cannot_escape_contract_working_directory(self): + manifest = base_manifest() + command = manifest["platforms"]["ios"]["verification"]["output"]["commands"][0] + command["command"] = ( + "xcodebuild -project ../APIExample-Audio/APIExample.xcodeproj " + "-scheme APIExample build" + ) + + self.assert_error_contains( + manifest, + "platforms.ios.verification.output.commands[0] is not a recognized ios build command", + ) + + def test_xcode_default_build_command_from_project_rules_is_accepted(self): + manifest = base_manifest() + command = manifest["platforms"]["macos"]["verification"]["output"]["commands"][0] + command["command"] = ( + "xcodebuild -workspace APIExample.xcworkspace -scheme APIExample " + "-configuration Release" + ) + + self.assertEqual(validate_manifest(manifest), []) + + def test_windows_build_pass_requires_windows_host_provenance(self): + manifest = base_manifest() + manifest["platforms"]["windows"]["verification"]["dispatch"]["host_platform"] = "darwin" + + self.assert_error_contains( + manifest, + "platforms.windows.verification.output.build_result=PASS requires host_platform=win32", + ) + + def test_rejects_platform_target_outside_its_platform_root(self): + manifest = base_manifest() + target = manifest["contract"]["output"]["platform_targets"]["android"] + target["target_project"] = "iOS/APIExample/" + target["files_allowed"] = ["iOS/APIExample/"] + + self.assert_error_contains( + manifest, + "contract.output.platform_targets.android.target_project must be inside Android/", + ) + + def test_rejects_platform_target_that_escapes_root_with_parent_segments(self): + manifest = base_manifest() + target = manifest["contract"]["output"]["platform_targets"]["android"] + target["target_project"] = "Android/../iOS/APIExample/" + target["files_allowed"] = ["Android/../iOS/APIExample/"] + + self.assert_error_contains( + manifest, + "contract.output.platform_targets.android.target_project must be inside Android/", + ) + + def test_rejects_dispatch_path_outside_workspace(self): + manifest = base_manifest() + manifest["platforms"]["android"]["implementation"]["dispatch"]["prompt"] = "../prompt.md" + + self.assert_error_contains(manifest, "must be relative to the execution workspace") + + def test_files_changed_accepts_deleted_path_inside_contract_scope(self): + manifest = base_manifest() + manifest["platforms"]["android"]["implementation"]["output"]["files_changed"] = [ + "Android/APIExample/deleted-case.java" + ] + + self.assertEqual(validate_manifest(manifest), []) + + def test_knowledge_update_requires_a_durable_file_change(self): + manifest = base_manifest() + for platform in PLATFORMS: + manifest["platforms"][platform]["implementation"]["output"]["files_changed"] = ["AGENTS.md"] + manifest["knowledge_updates"] = [ + { + "source": "pilot", + "impact_platforms": PLATFORMS, + "symptom": "Build blocked", + "root_cause": "Host mismatch", + "guardrail": "Use target CI", + "verification": "CI build", + "updated_at": "2026-07-10", + } + ] + + self.assert_error_contains(manifest, "knowledge_updates require at least one durable knowledge") + + def test_durable_knowledge_accepts_both_skill_directory_conventions(self): + self.assertTrue(is_durable_knowledge_path(".agent/skills/review-case/SKILL.md")) + self.assertTrue( + is_durable_knowledge_path( + "Android/APIExample/.agents/skills/review-case/SKILL.md" + ) + ) + + def test_evidence_files_are_rehashed_from_manifest_directory(self): + manifest = base_manifest() + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + artifacts = [manifest["contract"]] + for platform in PLATFORMS: + artifacts.extend(manifest["platforms"][platform].values()) + for index, item in enumerate(artifacts): + dispatch = item["dispatch"] + for path_field, hash_field in [ + ("prompt", "prompt_sha256"), + ("artifact", "artifact_sha256"), + ("input_snapshot", "input_snapshot_sha256"), + ("command_log", "command_log_sha256"), + ("repository_delta", "repository_delta_sha256"), + ]: + if path_field not in dispatch: + continue + path = root / dispatch[path_field] + path.parent.mkdir(parents=True, exist_ok=True) + if path_field == "repository_delta": + content = json.dumps( + {"changed_files": item["output"]["files_changed"]} + ).encode() + else: + content = f"{index}:{path_field}\n".encode() + path.write_bytes(content) + dispatch[hash_field] = hashlib.sha256(content).hexdigest() + + self.assertEqual(validate_evidence_files(manifest, root), []) + prompt = root / manifest["contract"]["dispatch"]["prompt"] + prompt.write_text("tampered\n", encoding="utf-8") + + errors = validate_evidence_files(manifest, root) + self.assertTrue(any("contract.dispatch.prompt_sha256 does not match" in error for error in errors)) + + def test_evidence_validation_rejects_delta_content_mismatch(self): + manifest = base_manifest() + implementation = manifest["platforms"]["android"]["implementation"] + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + delta_path = root / implementation["dispatch"]["repository_delta"] + delta_path.parent.mkdir(parents=True) + delta_path.write_text( + json.dumps({"changed_files": ["Android/APIExample/other.java"]}), + encoding="utf-8", + ) + implementation["dispatch"]["repository_delta_sha256"] = hashlib.sha256( + delta_path.read_bytes() + ).hexdigest() + + errors = validate_evidence_files(manifest, root) + + self.assertTrue( + any("repository_delta changed_files do not match" in error for error in errors), + errors, + ) + + +if __name__ == "__main__": + unittest.main() From bd6b480abad7c5d6784a872d9ec5815dc55f71a8 Mon Sep 17 00:00:00 2001 From: zhangwei Date: Fri, 17 Jul 2026 15:25:40 +0800 Subject: [PATCH 2/3] refactor(ai-engineering): support per-platform SDK targets --- .../api-example-release-iteration/SKILL.md | 7 +- docs/ai-engineering/release-iteration-gate.md | 9 +- docs/ai-engineering/templates/README.md | 2 +- .../acceptance-manifest-template.json | 22 +++-- .../templates/release-dry-run-template.md | 2 +- docs/ai-engineering/tools/README.md | 8 +- .../tools/orchestrate_case_execution.py | 12 ++- .../tools/orchestrate_case_execution_test.py | 42 ++++++++- .../tools/prepare_case_execution.py | 45 ++++++++-- .../tools/prepare_case_execution_test.py | 88 ++++++++++++++++++- .../tools/validate_acceptance_manifest.py | 51 ++++++++--- .../validate_acceptance_manifest_test.py | 45 ++++++++-- 12 files changed, 282 insertions(+), 51 deletions(-) diff --git a/.agent/skills/api-example-release-iteration/SKILL.md b/.agent/skills/api-example-release-iteration/SKILL.md index ec7433f0e..9213cbd36 100644 --- a/.agent/skills/api-example-release-iteration/SKILL.md +++ b/.agent/skills/api-example-release-iteration/SKILL.md @@ -15,7 +15,7 @@ Identify before editing: - New case, existing case update, docs-only, or SDK-version work. - Reference case for behavior and parity. - Verification budget and available platform hosts. -- Target RTC SDK version. +- Target RTC SDK version for each platform. Platforms may remain on different release lines. Android, iOS, macOS, and Windows are required by default. Contract may choose the appropriate full/audio/Compose/UIKit/Objective-C project for each platform; marking a platform not required needs an explicit waiver reason. @@ -53,10 +53,13 @@ Initialize one requirement: python3 docs/ai-engineering/tools/orchestrate_case_execution.py init \ --matrix docs/ai-engineering/case-maintenance-matrix.md \ --feature "" \ - --target-sdk-version "" \ + --target-sdk-version "4.6.2" \ + --platform-sdk-version "android=4.6.3" \ --run-dir /tmp/api-example-requirement ``` +`--target-sdk-version` supplies the baseline for all four platforms. Repeat `--platform-sdk-version platform=x.y.z` only for platforms whose target differs. The execution package and manifest store the resolved four-platform mapping, so staggered platform releases never require script or repository-profile changes. + The default `docs/ai-engineering/repository-profile.json` supplies repository-specific SDK package names and version sources. Use `--repository-profile` only for a checked-in alternative profile. The execution package records its repository-relative path and SHA-256. For a new requirement not yet actionable in the matrix, also pass `--sdk-family ""` and repeat `--key-api ""` as needed. diff --git a/docs/ai-engineering/release-iteration-gate.md b/docs/ai-engineering/release-iteration-gate.md index 30753ae37..6b0101ab1 100644 --- a/docs/ai-engineering/release-iteration-gate.md +++ b/docs/ai-engineering/release-iteration-gate.md @@ -15,7 +15,7 @@ Contract runs once. In a shared checkout, Android/iOS/macOS/Windows Implementati ## Intake Gate -Before Contract starts, identify feature behavior, SDK family, key APIs, target SDK version, reference case, expected user flow, repository scope, and available target verification hosts. Confirm that the checked-in repository profile describes the current SDK package names and version sources. All four top-level platforms are required by default. Contract may choose the appropriate project variant per platform. +Before Contract starts, identify feature behavior, SDK family, key APIs, target SDK version for each platform, reference case, expected user flow, repository scope, and available target verification hosts. Platform targets may differ when SDK release lines are staggered. Confirm that the checked-in repository profile describes the current SDK package names and version sources. All four top-level platforms are required by default. Contract may choose the appropriate project variant per platform. ## Contract Gate @@ -72,7 +72,7 @@ Any required platform blocker or cross-platform result other than `PASS` forces ## Release Checklist -Release is mandatory manifest data, not an agent. `requirement.target_sdk_version` and `release.target_sdk_version` must match. The orchestrator refreshes Android, iOS, macOS, and Windows dependency versions from live repository files during assembly. Non-`BLOCKED` acceptance requires all SDK-version checks to pass and permits no skipped repository release checks. +Release is mandatory manifest data, not an agent. `requirement.target_sdk_versions` and `release.target_sdk_versions` must contain the same Android, iOS, macOS, and Windows mapping. The orchestrator compares each platform only with its own target while refreshing dependency versions from live repository files during assembly. Non-`BLOCKED` acceptance requires all SDK-version checks to pass and permits no skipped repository release checks. A platform remaining on an older SDK is valid and does not require changing the script or repository profile. Jenkins packaging, QA validation, artifact URLs, and external website publication are downstream processes outside this repository workflow. A final `PASS` means the API Example source and repository checks are ready for that external handoff; the manifest does not represent packaging or QA completion. @@ -107,7 +107,8 @@ The validator enforces platform completeness, unique agent/run provenance, Contr python3 docs/ai-engineering/tools/orchestrate_case_execution.py init \ --matrix docs/ai-engineering/case-maintenance-matrix.md \ --feature "" \ - --target-sdk-version "" \ + --target-sdk-version "4.6.2" \ + --platform-sdk-version "android=4.6.3" \ --run-dir /tmp/api-example-requirement python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ @@ -120,7 +121,7 @@ python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ --run-dir /tmp/api-example-requirement --phase verification --model "" ``` -If the product requirement is not yet an actionable matrix row, initialize it directly with `--sdk-family` and one or more `--key-api` arguments. +`--target-sdk-version` is the baseline for all four platforms. Repeat `--platform-sdk-version platform=x.y.z` for each differing platform; omit overrides when all four use one version. If the product requirement is not yet an actionable matrix row, initialize it directly with `--sdk-family` and one or more `--key-api` arguments. Omitting `--platform` dispatches all four platform roles. Implementation runs are serialized in the shared checkout and reconciled after each run; Verification runs execute concurrently. Each independent `codex exec` run has a timeout, host provenance, a target-project working directory, and a hashed JSONL log. The execution package and input snapshots bind the checked-in repository profile by path and SHA-256. A prior `FAIL`/`BLOCKED` platform artifact can be replaced with `--retry`. diff --git a/docs/ai-engineering/templates/README.md b/docs/ai-engineering/templates/README.md index 54079c830..ab62636a2 100644 --- a/docs/ai-engineering/templates/README.md +++ b/docs/ai-engineering/templates/README.md @@ -4,7 +4,7 @@ This directory stores reusable templates for repository-level AI engineering wor Templates are not completed evidence. Before using a template, replace every placeholder with live repository data and current command output. -Templates may intentionally fail validation until their placeholders are replaced. The version 4 manifest stores one shared Contract and one Implementation/Verification pair for each official platform; use the orchestrator to add model, run, host, prompt/artifact, input-snapshot, working-directory, repository-delta, repository-profile, and hashed command-log provenance. Release data is limited to the target SDK version and repository checks. +Templates may intentionally fail validation until their placeholders are replaced. The version 4 manifest stores one shared Contract and one Implementation/Verification pair for each official platform; use the orchestrator to add model, run, host, prompt/artifact, input-snapshot, working-directory, repository-delta, repository-profile, and hashed command-log provenance. Release data is limited to per-platform target SDK versions and repository checks. Do not commit routine execution output from these templates unless the user explicitly asks for a repository evidence snapshot or the run is selected as a representative workflow validation example. diff --git a/docs/ai-engineering/templates/acceptance-manifest-template.json b/docs/ai-engineering/templates/acceptance-manifest-template.json index 9b78562c8..d92e990a0 100644 --- a/docs/ai-engineering/templates/acceptance-manifest-template.json +++ b/docs/ai-engineering/templates/acceptance-manifest-template.json @@ -5,7 +5,12 @@ "feature": "", "sdk_family": "", "key_apis": [""], - "target_sdk_version": "" + "target_sdk_versions": { + "android": "", + "ios": "", + "macos": "", + "windows": "" + } }, "contract": { "agent_id": "contract-agent-pending", @@ -232,12 +237,17 @@ }, "release": { "required": true, - "target_sdk_version": "", + "target_sdk_versions": { + "android": "", + "ios": "", + "macos": "", + "windows": "" + }, "checks": [ { "name": "sdk-version-android", "result": "BLOCKED", - "expected_version": "", + "expected_version": "", "actual_versions": {}, "evidence": "", "reason": "Pending live Android SDK version check." @@ -245,7 +255,7 @@ { "name": "sdk-version-ios", "result": "BLOCKED", - "expected_version": "", + "expected_version": "", "actual_versions": {}, "evidence": "", "reason": "Pending live iOS SDK version check." @@ -253,7 +263,7 @@ { "name": "sdk-version-macos", "result": "BLOCKED", - "expected_version": "", + "expected_version": "", "actual_versions": {}, "evidence": "", "reason": "Pending live macOS SDK version check." @@ -261,7 +271,7 @@ { "name": "sdk-version-windows", "result": "BLOCKED", - "expected_version": "", + "expected_version": "", "actual_versions": {}, "evidence": "", "reason": "Pending live Windows SDK version check." diff --git a/docs/ai-engineering/templates/release-dry-run-template.md b/docs/ai-engineering/templates/release-dry-run-template.md index c8ca2b8fe..0439c82cc 100644 --- a/docs/ai-engineering/templates/release-dry-run-template.md +++ b/docs/ai-engineering/templates/release-dry-run-template.md @@ -119,7 +119,7 @@ Testing: Release: - Required: -- Target SDK version: +- Target SDK versions by platform: - Checks: - Skipped checks: - External handoff: Jenkins packaging and QA diff --git a/docs/ai-engineering/tools/README.md b/docs/ai-engineering/tools/README.md index f08dcccd8..d6127c98a 100644 --- a/docs/ai-engineering/tools/README.md +++ b/docs/ai-engineering/tools/README.md @@ -18,12 +18,15 @@ Provider model names remain runtime inputs through `--model` or `CODEX_MODEL_STA python3 docs/ai-engineering/tools/orchestrate_case_execution.py init \ --matrix docs/ai-engineering/case-maintenance-matrix.md \ --feature "Join channel audio" \ - --target-sdk-version "4.6.4" \ + --target-sdk-version "4.6.2" \ + --platform-sdk-version "android=4.6.3" \ --run-dir /tmp/api-example-requirement ``` For a new feature outside the matrix backlog, add `--sdk-family "Full RTC" --key-api ""`; repeat `--key-api` for multiple APIs. +`--target-sdk-version` is a baseline expanded to Android, iOS, macOS, and Windows. Repeat `--platform-sdk-version platform=x.y.z` for any platform on a different SDK line. For example, the command above resolves to Android 4.6.3 and iOS/macOS/Windows 4.6.2. The final mapping is stored in the execution package and manifest; platform release cadence changes do not require edits to Python or the repository profile. + The default `docs/ai-engineering/repository-profile.json` defines repository-specific SDK version sources. Shared tools support Gradle property, CocoaPods package, and SDK archive-name sources without hard-coding a distribution in Python. `init` accepts `--repository-profile` for a checked-in alternative and binds its path and SHA-256 into the execution package. Dispatch Contract, then the two platform phases: @@ -82,7 +85,8 @@ python3 docs/ai-engineering/tools/orchestrate_case_execution.py dispatch \ ```bash python3 docs/ai-engineering/tools/prepare_case_execution.py \ --feature "Join channel audio" \ - --target-sdk-version "4.6.4" \ + --target-sdk-version "4.6.2" \ + --platform-sdk-version "android=4.6.3" \ --repository-profile docs/ai-engineering/repository-profile.json python3 docs/ai-engineering/tools/validate_acceptance_manifest.py ``` diff --git a/docs/ai-engineering/tools/orchestrate_case_execution.py b/docs/ai-engineering/tools/orchestrate_case_execution.py index a73b353d9..5429945d2 100644 --- a/docs/ai-engineering/tools/orchestrate_case_execution.py +++ b/docs/ai-engineering/tools/orchestrate_case_execution.py @@ -189,6 +189,7 @@ def init_workspace(args): sdk_family=args.sdk_family, key_apis=args.key_api, target_sdk_version=args.target_sdk_version, + platform_sdk_versions=args.platform_sdk_version, repository_profile=profile_path, ) package["matrix_path"] = stable_matrix_path(matrix_path) @@ -262,7 +263,8 @@ def write_prompt(run_dir, name, role, platform, package, routing, dependencies): Requirement: {requirement['feature']} SDK family: {requirement['sdk_family']} Key APIs: {', '.join(requirement['key_apis'])} -Target SDK version: {requirement['target_sdk_version']} +Target SDK versions: +{json.dumps(requirement['target_sdk_versions'], indent=2, ensure_ascii=False)} Role: {role} Platform: {platform or 'shared'} Target: {target} @@ -1314,7 +1316,7 @@ def assemble_workspace(args): "differences": args.cross_platform_difference, } manifest["release"]["checks"] = collect_sdk_version_checks( - manifest["requirement"]["target_sdk_version"], profile_path=profile_path + manifest["requirement"]["target_sdk_versions"], profile_path=profile_path ) errors = validate_manifest(manifest) errors.extend(validate_evidence_files(manifest, run_dir)) @@ -1537,6 +1539,12 @@ def main(argv=None): init_parser.add_argument("--sdk-family", help="Required for a feature outside the matrix backlog") init_parser.add_argument("--key-api", action="append", help="Key SDK API; repeat as needed") init_parser.add_argument("--target-sdk-version", required=True) + init_parser.add_argument( + "--platform-sdk-version", + action="append", + default=[], + help="Override one platform as platform=x.y.z; repeat as needed", + ) init_parser.add_argument("--run-dir", required=True) init_parser.add_argument("--index", type=int, default=0) init_parser.add_argument("--routing-config", default=str(DEFAULT_ROUTING_CONFIG)) diff --git a/docs/ai-engineering/tools/orchestrate_case_execution_test.py b/docs/ai-engineering/tools/orchestrate_case_execution_test.py index 889804002..07875122d 100644 --- a/docs/ai-engineering/tools/orchestrate_case_execution_test.py +++ b/docs/ai-engineering/tools/orchestrate_case_execution_test.py @@ -55,7 +55,10 @@ def run_orchestrator(self, *args): check=False, ) - def init_workspace(self, matrix_path, run_dir): + def init_workspace(self, matrix_path, run_dir, platform_sdk_versions=None): + version_args = [] + for value in platform_sdk_versions or []: + version_args.extend(["--platform-sdk-version", value]) result = self.run_orchestrator( "init", "--matrix", @@ -64,6 +67,7 @@ def init_workspace(self, matrix_path, run_dir): "Join channel audio", "--target-sdk-version", self.TARGET_SDK_VERSION, + *version_args, "--run-dir", str(run_dir), ) @@ -292,6 +296,37 @@ def test_init_accepts_new_requirement_outside_the_matrix_backlog(self): contract_prompt = (run_dir / "role-prompts/contract.md").read_text() self.assertNotIn("If Contract marks this platform required=false", contract_prompt) + def test_init_persists_and_prompts_staggered_platform_sdk_versions(self): + matrix_path = self.write_matrix() + with tempfile.TemporaryDirectory() as tmpdir: + run_dir = Path(tmpdir) / "run" + result = self.run_orchestrator( + "init", + "--matrix", + str(matrix_path), + "--feature", + "Join channel audio", + "--target-sdk-version", + "4.6.2", + "--platform-sdk-version", + "android=4.6.3", + "--run-dir", + str(run_dir), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + package = json.loads((run_dir / "execution-package.json").read_text()) + expected = { + "android": "4.6.3", + "ios": "4.6.2", + "macos": "4.6.2", + "windows": "4.6.2", + } + self.assertEqual(package["requirement"]["target_sdk_versions"], expected) + contract_prompt = (run_dir / "role-prompts/contract.md").read_text() + self.assertIn('"android": "4.6.3"', contract_prompt) + self.assertIn('"windows": "4.6.2"', contract_prompt) + def test_repository_matrix_identity_is_portable(self): identity = orchestrator.stable_matrix_path( REPO_ROOT / "docs/ai-engineering/case-maintenance-matrix.md" @@ -964,7 +999,10 @@ def test_full_multi_platform_run_assembles_blocked_v4_manifest(self): self.assertEqual(final_manifest["cross_platform_acceptance"]["result"], "BLOCKED") self.assertEqual(sorted(final_manifest["platforms"]), PLATFORMS) self.assertTrue(final_manifest["release"]["required"]) - self.assertEqual(final_manifest["release"]["target_sdk_version"], "4.6.4") + self.assertEqual( + final_manifest["release"]["target_sdk_versions"], + {platform: "4.6.4" for platform in PLATFORMS}, + ) validation = subprocess.run( [sys.executable, str(VALIDATOR), str(final_path)], cwd=REPO_ROOT, diff --git a/docs/ai-engineering/tools/prepare_case_execution.py b/docs/ai-engineering/tools/prepare_case_execution.py index dc729a7b2..8d0223036 100644 --- a/docs/ai-engineering/tools/prepare_case_execution.py +++ b/docs/ai-engineering/tools/prepare_case_execution.py @@ -18,6 +18,7 @@ "archive-name": {"path", "kind", "prefix", "suffix"}, } SEMVER_CAPTURE = r"([0-9]+\.[0-9]+\.[0-9]+)" +SDK_VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") DEFAULT_PLATFORM_TARGETS = { "android": "Android/APIExample/", "ios": "iOS/APIExample/", @@ -182,6 +183,28 @@ def load_repository_profile(profile_path=DEFAULT_REPOSITORY_PROFILE): return profile +def resolve_target_sdk_versions(target_sdk_version, platform_sdk_versions=None): + if not target_sdk_version: + raise ValueError("target_sdk_version is required") + if not SDK_VERSION_RE.fullmatch(str(target_sdk_version)): + raise ValueError("target_sdk_version must use x.y.z format") + versions = {platform: target_sdk_version for platform in PLATFORMS} + seen = set() + for value in platform_sdk_versions or []: + if not isinstance(value, str) or value.count("=") != 1: + raise ValueError("platform SDK version override must use platform=x.y.z") + platform, version = value.split("=", 1) + if platform not in PLATFORMS: + raise ValueError(f"unknown platform: {platform}") + if platform in seen: + raise ValueError(f"duplicate platform SDK version: {platform}") + if not SDK_VERSION_RE.fullmatch(version): + raise ValueError(f"{platform} SDK version must use x.y.z format") + seen.add(platform) + versions[platform] = version + return versions + + def sdk_version_pattern(source): kind = source["kind"] if kind == "gradle-property": @@ -195,7 +218,7 @@ def sdk_version_pattern(source): def collect_sdk_version_checks( - target_sdk_version, + target_sdk_versions, repo_root=REPO_ROOT, sources=None, profile_path=DEFAULT_REPOSITORY_PROFILE, @@ -204,6 +227,7 @@ def collect_sdk_version_checks( sources = load_repository_profile(profile_path)["sdk_version_sources"] checks = [] for platform in PLATFORMS: + target_sdk_version = target_sdk_versions[platform] entries = sources[platform] actual_versions = {} problems = [] @@ -303,9 +327,9 @@ def build_manifest_seed(requirement, source_case, version_sources): }, "release": { "required": True, - "target_sdk_version": requirement["target_sdk_version"], + "target_sdk_versions": dict(requirement["target_sdk_versions"]), "checks": collect_sdk_version_checks( - requirement["target_sdk_version"], sources=version_sources + requirement["target_sdk_versions"], sources=version_sources ), "skipped_checks": [], }, @@ -337,10 +361,12 @@ def prepare_case_execution( sdk_family=None, key_apis=None, target_sdk_version=None, + platform_sdk_versions=None, repository_profile=DEFAULT_REPOSITORY_PROFILE, ): - if not target_sdk_version: - raise ValueError("target_sdk_version is required") + target_sdk_versions = resolve_target_sdk_versions( + target_sdk_version, platform_sdk_versions + ) profile = load_repository_profile(repository_profile) backlog = generate_execution_units(matrix_path) matching = [ @@ -355,7 +381,7 @@ def prepare_case_execution( "feature": feature or selected["feature"], "sdk_family": sdk_family or selected["sdk_family"], "key_apis": key_apis or selected["key_apis"], - "target_sdk_version": target_sdk_version, + "target_sdk_versions": target_sdk_versions, } source_candidate = None if selected and selected["reference_candidates"]: @@ -395,6 +421,12 @@ def main(argv=None): parser.add_argument("--sdk-family", help="Required when the feature is not actionable in the matrix") parser.add_argument("--key-api", action="append", help="Key SDK API; repeat for multiple APIs") parser.add_argument("--target-sdk-version", required=True) + parser.add_argument( + "--platform-sdk-version", + action="append", + default=[], + help="Override one platform as platform=x.y.z; repeat as needed", + ) parser.add_argument("--repository-profile", default=str(DEFAULT_REPOSITORY_PROFILE)) parser.add_argument("--index", type=int, default=0) parser.add_argument("--output") @@ -407,6 +439,7 @@ def main(argv=None): sdk_family=args.sdk_family, key_apis=args.key_api, target_sdk_version=args.target_sdk_version, + platform_sdk_versions=args.platform_sdk_version, repository_profile=Path(args.repository_profile), ) except (OSError, ValueError) as exc: diff --git a/docs/ai-engineering/tools/prepare_case_execution_test.py b/docs/ai-engineering/tools/prepare_case_execution_test.py index 3ee33b3d5..ea50dcd24 100644 --- a/docs/ai-engineering/tools/prepare_case_execution_test.py +++ b/docs/ai-engineering/tools/prepare_case_execution_test.py @@ -14,6 +14,7 @@ collect_sdk_version_checks, load_repository_profile, prepare_case_execution, + resolve_target_sdk_versions, ) from validate_acceptance_manifest import validate_manifest @@ -24,6 +25,11 @@ class PrepareCaseExecutionTest(unittest.TestCase): TARGET_SDK_VERSION = "4.6.4" + def target_sdk_versions(self, **overrides): + versions = {platform: self.TARGET_SDK_VERSION for platform in PLATFORMS} + versions.update(overrides) + return versions + def write_matrix(self): handle = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) handle.write( @@ -59,9 +65,15 @@ def test_prepares_one_requirement_with_four_platform_delivery_units(self): self.assertNotIn("roles", manifest) self.assertEqual(sorted(package["role_contracts"]), ["contract", "implementation", "verification"]) self.assertEqual(sorted(manifest["contract"]["output"]["platform_targets"]), PLATFORMS) - self.assertEqual(manifest["requirement"]["target_sdk_version"], self.TARGET_SDK_VERSION) + self.assertEqual( + manifest["requirement"]["target_sdk_versions"], + self.target_sdk_versions(), + ) self.assertTrue(manifest["release"]["required"]) - self.assertEqual(manifest["release"]["target_sdk_version"], self.TARGET_SDK_VERSION) + self.assertEqual( + manifest["release"]["target_sdk_versions"], + self.target_sdk_versions(), + ) self.assertNotIn("qa_acceptance", manifest["release"]) self.assertNotIn("publication_channel", manifest["requirement"]) self.assertNotIn("publication", manifest["release"]) @@ -119,6 +131,44 @@ def test_requires_target_sdk_version(self): with self.assertRaisesRegex(ValueError, "target_sdk_version is required"): prepare_case_execution(self.write_matrix(), feature="Join channel audio") + def test_platform_sdk_version_overrides_the_shared_baseline(self): + package = prepare_case_execution( + self.write_matrix(), + feature="Join channel audio", + target_sdk_version="4.6.2", + platform_sdk_versions=["android=4.6.3"], + ) + + expected = self.target_sdk_versions( + android="4.6.3", ios="4.6.2", macos="4.6.2", windows="4.6.2" + ) + manifest = package["acceptance_manifest_seed"] + self.assertEqual(package["requirement"]["target_sdk_versions"], expected) + self.assertEqual(manifest["release"]["target_sdk_versions"], expected) + self.assertEqual( + { + check["name"]: check["expected_version"] + for check in manifest["release"]["checks"] + }, + {f"sdk-version-{platform}": version for platform, version in expected.items()}, + ) + + def test_rejects_invalid_platform_sdk_version_overrides(self): + cases = [ + (["android"], "must use platform=x.y.z"), + (["linux=4.6.3"], "unknown platform: linux"), + (["android=next"], "android SDK version must use x.y.z format"), + (["android=4.6.3", "android=4.6.4"], "duplicate platform SDK version: android"), + ] + + for overrides, message in cases: + with self.subTest(overrides=overrides), self.assertRaisesRegex(ValueError, message): + resolve_target_sdk_versions("4.6.2", overrides) + + def test_rejects_invalid_baseline_sdk_version(self): + with self.assertRaisesRegex(ValueError, "target_sdk_version must use x.y.z format"): + resolve_target_sdk_versions("next") + def test_collects_live_sdk_version_evidence(self): for distribution in ["Shengwang", "Agora"]: with self.subTest(distribution=distribution), tempfile.TemporaryDirectory() as tmpdir: @@ -127,7 +177,7 @@ def test_collects_live_sdk_version_evidence(self): profile = load_repository_profile(profile_path) checks = collect_sdk_version_checks( - self.TARGET_SDK_VERSION, + self.target_sdk_versions(), repo_root=root, sources=profile["sdk_version_sources"], ) @@ -146,7 +196,7 @@ def test_collects_live_sdk_version_evidence(self): encoding="utf-8", ) checks = collect_sdk_version_checks( - self.TARGET_SDK_VERSION, + self.target_sdk_versions(), repo_root=root, profile_path=profile_path, ) @@ -156,6 +206,36 @@ def test_collects_live_sdk_version_evidence(self): self.assertEqual(windows["result"], "BLOCKED") self.assertIn("4.6.2", windows["reason"]) + def test_collects_staggered_platform_sdk_versions(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + profile_path = self.write_repository_profile(root, "Agora") + (root / "Android/gradle.properties").write_text( + "rtc_sdk_version = 4.6.3\n", encoding="utf-8" + ) + for path_text in ["iOS/Podfile", "macOS/Podfile", "windows/install.ps1"]: + path = root / path_text + path.write_text( + path.read_text(encoding="utf-8").replace("4.6.4", "4.6.2"), + encoding="utf-8", + ) + + targets = { + "android": "4.6.3", + "ios": "4.6.2", + "macos": "4.6.2", + "windows": "4.6.2", + } + checks = collect_sdk_version_checks( + targets, repo_root=root, profile_path=profile_path + ) + + self.assertTrue(all(check["result"] == "PASS" for check in checks)) + self.assertEqual( + {check["name"]: check["expected_version"] for check in checks}, + {f"sdk-version-{platform}": version for platform, version in targets.items()}, + ) + def write_repository_profile(self, root, distribution): sources = { "android": [ diff --git a/docs/ai-engineering/tools/validate_acceptance_manifest.py b/docs/ai-engineering/tools/validate_acceptance_manifest.py index 536c71f9f..9207a6267 100644 --- a/docs/ai-engineering/tools/validate_acceptance_manifest.py +++ b/docs/ai-engineering/tools/validate_acceptance_manifest.py @@ -34,8 +34,8 @@ "release", "knowledge_updates", } -REQUIREMENT_FIELDS = {"feature", "sdk_family", "key_apis", "target_sdk_version"} -RELEASE_FIELDS = {"required", "target_sdk_version", "checks", "skipped_checks"} +REQUIREMENT_FIELDS = {"feature", "sdk_family", "key_apis", "target_sdk_versions"} +RELEASE_FIELDS = {"required", "target_sdk_versions", "checks", "skipped_checks"} CONTRACT_OUTPUT_FIELDS = { "scenario", "key_apis", @@ -103,6 +103,20 @@ def validate_no_placeholders(value, path, errors): validate_no_placeholders(child, f"{path}[{index}]", errors) +def validate_target_sdk_versions(value, path, errors): + if not isinstance(value, dict): + errors.append(f"{path} must be an object") + return + if set(value) != set(PLATFORMS): + errors.append( + f"{path} must define exactly " + ", ".join(PLATFORMS) + ) + for platform in PLATFORMS: + version = value.get(platform) + if not SDK_VERSION_RE.fullmatch(str(version or "")): + errors.append(f"{path}.{platform} must use x.y.z format") + + def validate_manifest_shape(manifest, errors): for key in sorted(REQUIRED_TOP_LEVEL - set(manifest)): errors.append(f"missing top-level field: {key}") @@ -121,11 +135,11 @@ def validate_manifest_shape(manifest, errors): errors.append(f"requirement.{field} is required") for field in sorted(set(requirement) - REQUIREMENT_FIELDS): errors.append(f"unsupported requirement field: {field}") - target_sdk_version = requirement.get("target_sdk_version") - if is_non_empty(target_sdk_version) and not SDK_VERSION_RE.fullmatch( - str(target_sdk_version) - ): - errors.append("requirement.target_sdk_version must use x.y.z format") + target_sdk_versions = requirement.get("target_sdk_versions") + if is_non_empty(target_sdk_versions): + validate_target_sdk_versions( + target_sdk_versions, "requirement.target_sdk_versions", errors + ) def validate_manifest(manifest): @@ -932,9 +946,16 @@ def validate_release(manifest, errors): for field in sorted(set(release) - RELEASE_FIELDS): errors.append(f"unsupported release field: {field}") requirement = manifest.get("requirement", {}) - target_sdk_version = requirement.get("target_sdk_version") - if release.get("target_sdk_version") != target_sdk_version: - errors.append("release.target_sdk_version must match requirement.target_sdk_version") + target_sdk_versions = requirement.get("target_sdk_versions") + release_target_sdk_versions = release.get("target_sdk_versions") + if is_non_empty(release_target_sdk_versions): + validate_target_sdk_versions( + release_target_sdk_versions, "release.target_sdk_versions", errors + ) + if release_target_sdk_versions != target_sdk_versions: + errors.append( + "release.target_sdk_versions must match requirement.target_sdk_versions" + ) checks = release.get("checks", []) if not checks: errors.append("release.checks must include SDK version checks") @@ -959,14 +980,20 @@ def validate_release(manifest, errors): if result in {"FAIL", "BLOCKED"} and not check.get("reason"): errors.append(f"release.checks[{index}].reason is required when result is {result}") if check.get("name") in expected_sdk_checks: - if check.get("expected_version") != target_sdk_version: + platform = check["name"].removeprefix("sdk-version-") + target_sdk_version = ( + target_sdk_versions.get(platform) + if isinstance(target_sdk_versions, dict) + else None + ) + if target_sdk_version and check.get("expected_version") != target_sdk_version: errors.append( f"{check.get('name')} expected_version must match {target_sdk_version}" ) actual_versions = check.get("actual_versions") if not isinstance(actual_versions, dict) or not actual_versions: errors.append(f"{check.get('name')} actual_versions must be a non-empty object") - elif result == "PASS" and any( + elif target_sdk_version and result == "PASS" and any( version != target_sdk_version for version in actual_versions.values() ): errors.append( diff --git a/docs/ai-engineering/tools/validate_acceptance_manifest_test.py b/docs/ai-engineering/tools/validate_acceptance_manifest_test.py index 464788e72..e37974f26 100644 --- a/docs/ai-engineering/tools/validate_acceptance_manifest_test.py +++ b/docs/ai-engineering/tools/validate_acceptance_manifest_test.py @@ -128,7 +128,7 @@ def base_manifest(): "feature": "Join channel audio", "sdk_family": "Full RTC", "key_apis": ["joinChannel", "setAudioProfile"], - "target_sdk_version": "4.6.4", + "target_sdk_versions": {platform: "4.6.4" for platform in PLATFORMS}, }, "contract": artifact( "contract", @@ -154,7 +154,7 @@ def base_manifest(): }, "release": { "required": True, - "target_sdk_version": "4.6.4", + "target_sdk_versions": {platform: "4.6.4" for platform in PLATFORMS}, "checks": [ { "name": f"sdk-version-{platform}", @@ -442,20 +442,47 @@ def test_rejects_removed_publication_fields(self): self.assert_error_contains(manifest, "unsupported requirement field: publication_channel") self.assert_error_contains(manifest, "unsupported release field: publication") - def test_release_target_version_must_match_requirement(self): + def test_release_target_versions_must_match_requirement(self): manifest = base_manifest() - manifest["release"]["target_sdk_version"] = "4.6.3" + manifest["release"]["target_sdk_versions"]["android"] = "4.6.3" self.assert_error_contains( - manifest, "release.target_sdk_version must match requirement.target_sdk_version" + manifest, "release.target_sdk_versions must match requirement.target_sdk_versions" ) - def test_target_sdk_version_requires_semver_triplet(self): + def test_target_sdk_versions_require_all_platform_semver_triplets(self): manifest = base_manifest() - manifest["requirement"]["target_sdk_version"] = "next" - manifest["release"]["target_sdk_version"] = "next" + manifest["requirement"]["target_sdk_versions"]["android"] = "next" + manifest["release"]["target_sdk_versions"]["android"] = "next" - self.assert_error_contains(manifest, "requirement.target_sdk_version must use x.y.z format") + self.assert_error_contains( + manifest, + "requirement.target_sdk_versions.android must use x.y.z format", + ) + + manifest = base_manifest() + manifest["requirement"]["target_sdk_versions"]["android"] = "" + manifest["release"]["target_sdk_versions"]["android"] = "" + self.assert_error_contains( + manifest, + "requirement.target_sdk_versions.android must use x.y.z format", + ) + + manifest = base_manifest() + del manifest["requirement"]["target_sdk_versions"]["windows"] + del manifest["release"]["target_sdk_versions"]["windows"] + self.assert_error_contains( + manifest, + "requirement.target_sdk_versions must define exactly android, ios, macos, windows", + ) + + manifest = base_manifest() + manifest["requirement"]["target_sdk_versions"]["linux"] = "4.6.4" + manifest["release"]["target_sdk_versions"]["linux"] = "4.6.4" + self.assert_error_contains( + manifest, + "requirement.target_sdk_versions must define exactly android, ios, macos, windows", + ) def test_release_sdk_check_requires_all_sources_to_match_target(self): manifest = base_manifest() From 2e20739d0356df108271a32a8f320bf6219d667b Mon Sep 17 00:00:00 2001 From: zhangwei Date: Fri, 17 Jul 2026 18:09:39 +0800 Subject: [PATCH 3/3] ci(ios): add signing preflight and unsigned build validation - validate Jenkins signing profiles and certificate expiration - support legacy and modern distribution certificate names - switch Azure iOS jobs to unsigned compile checks --- .github/ci/build/build_ios.sh | 4 + .github/ci/build/check_ios_signing_assets.py | 343 ++++++++++++++++++ .../tests/test_check_ios_signing_assets.py | 205 +++++++++++ cicd/build-template/build-ios.yml | 20 +- cicd/scripts/ios_build.sh | 95 ++--- 5 files changed, 577 insertions(+), 90 deletions(-) create mode 100644 .github/ci/build/check_ios_signing_assets.py create mode 100644 .github/ci/build/tests/test_check_ios_signing_assets.py diff --git a/.github/ci/build/build_ios.sh b/.github/ci/build/build_ios.sh index 96b54690d..234d96c30 100644 --- a/.github/ci/build/build_ios.sh +++ b/.github/ci/build/build_ios.sh @@ -152,6 +152,10 @@ if [ $compress_apiexample = true ]; then fi if [ $compile_project = true ]; then + if ! python3 ./.github/ci/build/check_ios_signing_assets.py \ + --export-options "./iOS/${ios_direction}/ExportOptions.plist"; then + echo "WARNING: iOS signing asset preflight could not complete; continuing packaging" + fi cd ./$unzip_name/samples/${ios_direction} ./cloud_build.sh || exit 1 cd - diff --git a/.github/ci/build/check_ios_signing_assets.py b/.github/ci/build/check_ios_signing_assets.py new file mode 100644 index 000000000..ede69a1e8 --- /dev/null +++ b/.github/ci/build/check_ios_signing_assets.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Validate the iOS signing assets that Jenkins will use for IPA export.""" + +from __future__ import annotations + +import argparse +import hashlib +import math +import os +import plistlib +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Set, Tuple + + +IDENTITY_PATTERN = re.compile( + r'^\s*\d+\)\s+([0-9A-Fa-f]{40})\s+"([^"]+)"' +) +DISTRIBUTION_CERTIFICATE_ALIASES = frozenset( + { + "apple distribution", + "ios distribution", + "iphone distribution", + } +) + + +@dataclass(frozen=True) +class ProvisioningProfile: + path: Path + name: str + team_ids: Tuple[str, ...] + expires_at: datetime + certificates: Dict[str, bytes] + + +def normalize_datetime(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def parse_identities(output: str) -> Dict[str, str]: + identities: Dict[str, str] = {} + for line in output.splitlines(): + match = IDENTITY_PATTERN.match(line) + if match: + identities[match.group(1).upper()] = match.group(2) + return identities + + +def certificate_selector_matches_identity(selector: str, identity_name: str) -> bool: + normalized_selector = selector.strip().casefold() + normalized_identity = identity_name.casefold() + if normalized_selector in normalized_identity: + return True + return normalized_selector in DISTRIBUTION_CERTIFICATE_ALIASES and any( + alias in normalized_identity for alias in DISTRIBUTION_CERTIFICATE_ALIASES + ) + + +def parse_certificate_expiration(output: str) -> datetime: + for line in output.splitlines(): + if line.startswith("notAfter="): + return normalize_datetime(parsedate_to_datetime(line.split("=", 1)[1])) + raise ValueError("openssl output does not contain notAfter") + + +def expiration_status( + expires_at: datetime, + now: datetime, + critical_days: int, + warning_days: int, +) -> Tuple[str, int]: + remaining_seconds = (normalize_datetime(expires_at) - normalize_datetime(now)).total_seconds() + days_left = math.floor(remaining_seconds / 86400) + if remaining_seconds <= 0: + return "expired", days_left + if remaining_seconds < critical_days * 86400: + return "critical", days_left + if remaining_seconds < warning_days * 86400: + return "warning", days_left + return "ok", days_left + + +def load_export_options(path: Path) -> Tuple[str, str, List[str]]: + with path.open("rb") as stream: + options = plistlib.load(stream) + + team_id = options.get("teamID") + certificate_selector = options.get("signingCertificate") + provisioning_profiles = options.get("provisioningProfiles") + if not isinstance(team_id, str) or not team_id: + raise ValueError("ExportOptions.plist does not define teamID") + if not isinstance(certificate_selector, str) or not certificate_selector: + raise ValueError("ExportOptions.plist does not define signingCertificate") + if not isinstance(provisioning_profiles, dict) or not provisioning_profiles: + raise ValueError("ExportOptions.plist does not define provisioningProfiles") + + profile_names = sorted( + {value for value in provisioning_profiles.values() if isinstance(value, str) and value} + ) + if not profile_names: + raise ValueError("ExportOptions.plist has no provisioning profile names") + return team_id, certificate_selector, profile_names + + +def profile_directories(explicit_directories: List[Path]) -> List[Path]: + if explicit_directories: + return explicit_directories + + configured = os.environ.get("IOS_PROVISIONING_PROFILE_DIRS") + if configured: + return [Path(value).expanduser() for value in configured.split(os.pathsep) if value] + + home = Path.home() + return [ + home / "Library/MobileDevice/Provisioning Profiles", + home / "Library/Developer/Xcode/UserData/Provisioning Profiles", + ] + + +def profile_files(directories: Iterable[Path]) -> List[Path]: + files: Set[Path] = set() + for directory in directories: + if not directory.is_dir(): + continue + files.update(directory.rglob("*.mobileprovision")) + files.update(directory.rglob("*.provisionprofile")) + return sorted(files) + + +def decode_profile(path: Path, security: str) -> Optional[ProvisioningProfile]: + result = subprocess.run( + [security, "cms", "-D", "-i", str(path)], + check=False, + capture_output=True, + ) + if result.returncode != 0: + return None + + try: + payload = plistlib.loads(result.stdout) + name = payload["Name"] + expires_at = normalize_datetime(payload["ExpirationDate"]) + except (KeyError, TypeError, ValueError, plistlib.InvalidFileException): + return None + + team_ids = payload.get("TeamIdentifier", ()) + if isinstance(team_ids, str): + team_ids = (team_ids,) + elif isinstance(team_ids, list): + team_ids = tuple(value for value in team_ids if isinstance(value, str)) + else: + team_ids = () + + certificates: Dict[str, bytes] = {} + for certificate in payload.get("DeveloperCertificates", ()): + if isinstance(certificate, bytes): + fingerprint = hashlib.sha1(certificate).hexdigest().upper() + certificates[fingerprint] = certificate + + if not isinstance(name, str) or not certificates: + return None + return ProvisioningProfile(path, name, team_ids, expires_at, certificates) + + +def installed_identities(security: str) -> Dict[str, str]: + result = subprocess.run( + [security, "find-identity", "-v", "-p", "codesigning"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or "security find-identity failed") + return parse_identities(result.stdout) + + +def certificate_expiration(certificate: bytes, openssl: str) -> datetime: + result = subprocess.run( + [openssl, "x509", "-inform", "DER", "-noout", "-enddate"], + input=certificate, + check=False, + capture_output=True, + ) + if result.returncode != 0: + message = result.stderr.decode(errors="replace").strip() + raise RuntimeError(message or "openssl could not read certificate") + return parse_certificate_expiration(result.stdout.decode(errors="replace")) + + +def report_expiration( + label: str, + expires_at: datetime, + now: datetime, + critical_days: int, + warning_days: int, +) -> None: + status, days_left = expiration_status( + expires_at, now, critical_days, warning_days + ) + print(f"{label} expires: {expires_at.isoformat()} ({days_left} days remaining)") + if status == "expired": + print(f"CRITICAL: {label} has expired", file=sys.stderr) + elif status == "critical": + print( + f"CRITICAL: {label} expires in less than {critical_days} days", + file=sys.stderr, + ) + elif status == "warning": + print(f"WARNING: {label} expires in less than {warning_days} days") + + +def validate(args: argparse.Namespace) -> int: + if args.critical_days < 0: + raise ValueError("critical-days must not be negative") + if args.warning_days < args.critical_days: + raise ValueError("warning-days must be greater than or equal to critical-days") + + security = shutil.which("security") + openssl = shutil.which("openssl") + if not security: + raise RuntimeError("macOS security command was not found") + if not openssl: + raise RuntimeError("openssl command was not found") + + team_id, certificate_selector, profile_names = load_export_options( + args.export_options + ) + directories = profile_directories(args.profile_directory) + profiles = [] + for path in profile_files(directories): + profile = decode_profile(path, security) + if profile is not None: + profiles.append(profile) + identities = installed_identities(security) + now = datetime.now(timezone.utc) + + print("iOS signing asset preflight") + print(f"Export options: {args.export_options}") + print(f"Team ID: {team_id}") + print(f"Certificate selector: {certificate_selector}") + + for profile_name in profile_names: + candidates = [ + profile + for profile in profiles + if profile.name == profile_name and team_id in profile.team_ids + ] + viable: List[Tuple[ProvisioningProfile, Set[str]]] = [] + for profile in candidates: + matching_fingerprints = { + fingerprint + for fingerprint in profile.certificates + if fingerprint in identities + and certificate_selector_matches_identity( + certificate_selector, identities[fingerprint] + ) + } + if matching_fingerprints: + viable.append((profile, matching_fingerprints)) + + if not viable: + searched = ", ".join(str(path) for path in directories) + print( + f"WARNING: no installed signing identity matches profile '{profile_name}' " + f"for team {team_id}; searched {searched}", + file=sys.stderr, + ) + continue + + profile, fingerprints = max(viable, key=lambda item: item[0].expires_at) + print(f"Provisioning profile: {profile.name} ({profile.path})") + report_expiration( + f"Provisioning profile '{profile.name}'", + profile.expires_at, + now, + args.critical_days, + args.warning_days, + ) + + certificate_options = [ + ( + certificate_expiration(profile.certificates[fingerprint], openssl), + fingerprint, + ) + for fingerprint in fingerprints + ] + expires_at, fingerprint = max(certificate_options) + print(f"Signing identity: {identities[fingerprint]}") + print(f"Certificate SHA-1: {fingerprint}") + report_expiration( + "Signing certificate", + expires_at, + now, + args.critical_days, + args.warning_days, + ) + + print("iOS signing asset preflight completed") + return 0 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--export-options", required=True, type=Path) + parser.add_argument( + "--profile-directory", + action="append", + default=[], + type=Path, + help="Provisioning profile directory; may be supplied more than once", + ) + parser.add_argument( + "--critical-days", + type=int, + default=int(os.environ.get("IOS_SIGNING_CRITICAL_DAYS", "30")), + ) + parser.add_argument( + "--warning-days", + type=int, + default=int(os.environ.get("IOS_SIGNING_WARNING_DAYS", "90")), + ) + return parser.parse_args() + + +def main() -> int: + try: + return validate(parse_args()) + except (OSError, RuntimeError, ValueError, plistlib.InvalidFileException) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/ci/build/tests/test_check_ios_signing_assets.py b/.github/ci/build/tests/test_check_ios_signing_assets.py new file mode 100644 index 000000000..d9304e0a5 --- /dev/null +++ b/.github/ci/build/tests/test_check_ios_signing_assets.py @@ -0,0 +1,205 @@ +import hashlib +import importlib.util +import plistlib +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + + +MODULE_PATH = Path(__file__).parents[1] / "check_ios_signing_assets.py" +SPEC = importlib.util.spec_from_file_location("check_ios_signing_assets", MODULE_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +class SigningAssetCheckTests(unittest.TestCase): + def test_load_export_options_deduplicates_profile_names(self): + options = { + "teamID": "TEAM123", + "signingCertificate": "Apple Distribution", + "provisioningProfiles": { + "example.app": "shared-profile", + "example.app.extension": "shared-profile", + }, + } + with tempfile.NamedTemporaryFile() as stream: + plistlib.dump(options, stream) + stream.flush() + team_id, selector, profiles = MODULE.load_export_options( + Path(stream.name) + ) + + self.assertEqual(team_id, "TEAM123") + self.assertEqual(selector, "Apple Distribution") + self.assertEqual(profiles, ["shared-profile"]) + + def test_parse_identities_returns_sha1_to_name_mapping(self): + output = ( + ' 1) 553B538240B4CEF55467E6515514766F6767692C ' + '"iPhone Distribution: Example (TEAM123)"\n' + " 1 valid identities found\n" + ) + + self.assertEqual( + MODULE.parse_identities(output), + { + "553B538240B4CEF55467E6515514766F6767692C": ( + "iPhone Distribution: Example (TEAM123)" + ) + }, + ) + + def test_certificate_selector_matches_distribution_aliases(self): + for identity_name in ( + "Apple Distribution: Example (TEAM123)", + "iOS Distribution: Example (TEAM123)", + "iPhone Distribution: Example (TEAM123)", + ): + with self.subTest(identity_name=identity_name): + self.assertTrue( + MODULE.certificate_selector_matches_identity( + "iPhone Distribution", identity_name + ) + ) + + self.assertFalse( + MODULE.certificate_selector_matches_identity( + "iPhone Distribution", "Apple Development: Example (TEAM123)" + ) + ) + + def test_decode_profile_records_embedded_certificate_fingerprint(self): + certificate = b"test certificate bytes" + payload = plistlib.dumps( + { + "Name": "shared-profile", + "TeamIdentifier": ["TEAM123"], + "ExpirationDate": datetime(2030, 1, 1), + "DeveloperCertificates": [certificate], + } + ) + completed = SimpleNamespace(returncode=0, stdout=payload, stderr=b"") + + with mock.patch.object(MODULE.subprocess, "run", return_value=completed): + profile = MODULE.decode_profile(Path("profile.mobileprovision"), "security") + + self.assertIsNotNone(profile) + fingerprint = hashlib.sha1(certificate).hexdigest().upper() + self.assertEqual(profile.certificates, {fingerprint: certificate}) + self.assertEqual(profile.team_ids, ("TEAM123",)) + + def test_expiration_status_is_critical_inside_critical_window(self): + now = datetime(2026, 1, 1, tzinfo=timezone.utc) + + self.assertEqual( + MODULE.expiration_status( + datetime(2026, 1, 21, tzinfo=timezone.utc), now, 30, 90 + )[0], + "critical", + ) + self.assertEqual( + MODULE.expiration_status( + datetime(2026, 3, 2, tzinfo=timezone.utc), now, 30, 90 + )[0], + "warning", + ) + self.assertEqual( + MODULE.expiration_status( + datetime(2026, 6, 1, tzinfo=timezone.utc), now, 30, 90 + )[0], + "ok", + ) + + def test_parse_certificate_expiration_reads_openssl_date(self): + expires_at = MODULE.parse_certificate_expiration( + "notAfter=Dec 11 02:30:30 2026 GMT\n" + ) + + self.assertEqual( + expires_at, + datetime(2026, 12, 11, 2, 30, 30, tzinfo=timezone.utc), + ) + + def test_validate_matches_profile_certificate_to_installed_identity(self): + certificate = b"matching distribution certificate" + fingerprint = hashlib.sha1(certificate).hexdigest().upper() + profile = MODULE.ProvisioningProfile( + path=Path("profile.mobileprovision"), + name="shared-profile", + team_ids=("TEAM123",), + expires_at=datetime(2030, 1, 1, tzinfo=timezone.utc), + certificates={fingerprint: certificate}, + ) + options = { + "teamID": "TEAM123", + "signingCertificate": "iPhone Distribution", + "provisioningProfiles": {"example.app": "shared-profile"}, + } + + with tempfile.NamedTemporaryFile() as stream: + plistlib.dump(options, stream) + stream.flush() + args = SimpleNamespace( + export_options=Path(stream.name), + profile_directory=[Path("profiles")], + critical_days=30, + warning_days=90, + ) + with mock.patch.object( + MODULE.shutil, "which", side_effect=lambda value: value + ), mock.patch.object( + MODULE, "profile_files", return_value=[profile.path] + ), mock.patch.object( + MODULE, "decode_profile", return_value=profile + ), mock.patch.object( + MODULE, + "installed_identities", + return_value={ + fingerprint: "Apple Distribution: Example (TEAM123)" + }, + ), mock.patch.object( + MODULE, + "certificate_expiration", + return_value=datetime(2030, 1, 1, tzinfo=timezone.utc), + ) as certificate_expiration: + result = MODULE.validate(args) + + self.assertEqual(result, 0) + certificate_expiration.assert_called_once_with(certificate, "openssl") + + def test_validate_missing_identity_only_warns(self): + options = { + "teamID": "TEAM123", + "signingCertificate": "Apple Distribution", + "provisioningProfiles": {"example.app": "shared-profile"}, + } + + with tempfile.NamedTemporaryFile() as stream: + plistlib.dump(options, stream) + stream.flush() + args = SimpleNamespace( + export_options=Path(stream.name), + profile_directory=[Path("profiles")], + critical_days=30, + warning_days=90, + ) + with mock.patch.object( + MODULE.shutil, "which", side_effect=lambda value: value + ), mock.patch.object( + MODULE, "profile_files", return_value=[] + ), mock.patch.object( + MODULE, "installed_identities", return_value={} + ): + result = MODULE.validate(args) + + self.assertEqual(result, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/cicd/build-template/build-ios.yml b/cicd/build-template/build-ios.yml index 89fd342a0..9a9ab7a62 100644 --- a/cicd/build-template/build-ios.yml +++ b/cicd/build-template/build-ios.yml @@ -16,26 +16,10 @@ jobs: steps: - script: cd 'cicd/scripts' && ls && python keycenter.py && ls + displayName: 'Generate KeyCenter' env: AGORA_APP_ID: $(agora.appId) File_Directory: '../../${{ parameters.workingDirectory }}/${{ parameters.project }}/Common' - - - task: InstallAppleCertificate@2 - inputs: - certSecureFile: 'App.p12' - certPwd: $(agora.password) - - - task: InstallAppleProvisioningProfile@1 - inputs: - provProfileSecureFile: 'App.mobileprovision' - script: cd 'cicd/scripts' && chmod +x ios_build.sh && ./ios_build.sh ../../${{ parameters.workingDirectory }} ${{ parameters.project }} ${{ parameters.scheme }} - - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: ${{ parameters.workingDirectory }}/app - ArtifactName: ${{ parameters.displayName }} - - - template: github-release.yml - parameters: - displayName: ${{ parameters.displayName }} + displayName: 'Build without code signing' diff --git a/cicd/scripts/ios_build.sh b/cicd/scripts/ios_build.sh index a32c91747..56d939c10 100755 --- a/cicd/scripts/ios_build.sh +++ b/cicd/scripts/ios_build.sh @@ -1,81 +1,32 @@ -WORKING_PATH=$1 -APP_Project=$2 -APP_TARGET=$3 +#!/usr/bin/env bash +set -euo pipefail + +WORKING_PATH="$1" +APP_PROJECT="$2" +APP_TARGET="$3" MODE=Release echo "WORKING_PATH: ${WORKING_PATH}" echo "APP_TARGET: ${APP_TARGET}" -cd ${WORKING_PATH} -echo `pwd` - -rm -f *.ipa -rm -rf *.app -rm -f *.zip -rm -rf dSYMs -rm -rf *.dSYM -rm -f *dSYMs.zip -rm -rf *.xcarchive - -Export_Plist_File=ExportOptions.plist - -BUILD_DATE=`date +%Y-%m-%d-%H.%M.%S` -ArchivePath=${APP_TARGET}-${BUILD_DATE}.xcarchive -PBXPROJ_PATH="${APP_Project}.xcodeproj/project.pbxproj" -echo PBXPROJ_PATH: $PBXPROJ_PATH - -# 主项目工程配置 -# Debug -/usr/libexec/PlistBuddy -c "Set :objects:03D13BF72448758C00B599B3:buildSettings:CODE_SIGN_STYLE 'Manual'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:03D13BF72448758C00B599B3:buildSettings:DEVELOPMENT_TEAM 'GM72UGLGZW'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:03D13BF72448758C00B599B3:buildSettings:PROVISIONING_PROFILE_SPECIFIER 'App'" $PBXPROJ_PATH -# Release -/usr/libexec/PlistBuddy -c "Set :objects:03D13BF82448758C00B599B3:buildSettings:CODE_SIGN_STYLE 'Manual'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:03D13BF82448758C00B599B3:buildSettings:DEVELOPMENT_TEAM 'GM72UGLGZW'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:03D13BF82448758C00B599B3:buildSettings:PROVISIONING_PROFILE_SPECIFIER 'App'" $PBXPROJ_PATH - -# 屏幕共享Extension -# Debug -/usr/libexec/PlistBuddy -c "Set :objects:0339BEB825205B80007D4FDD:buildSettings:CODE_SIGN_STYLE 'Manual'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:0339BEB825205B80007D4FDD:buildSettings:DEVELOPMENT_TEAM 'GM72UGLGZW'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:0339BEB825205B80007D4FDD:buildSettings:PROVISIONING_PROFILE_SPECIFIER 'App'" $PBXPROJ_PATH -# Release -/usr/libexec/PlistBuddy -c "Set :objects:0339BEB925205B80007D4FDD:buildSettings:CODE_SIGN_STYLE 'Manual'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:0339BEB925205B80007D4FDD:buildSettings:DEVELOPMENT_TEAM 'GM72UGLGZW'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:0339BEB925205B80007D4FDD:buildSettings:PROVISIONING_PROFILE_SPECIFIER 'App'" $PBXPROJ_PATH +cd "${WORKING_PATH}" +pwd -# SimpleFilter -# Debug -/usr/libexec/PlistBuddy -c "Set :objects:8B10BE1726AFFFA6002E1373:buildSettings:CODE_SIGN_STYLE 'Manual'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:8B10BE1726AFFFA6002E1373:buildSettings:DEVELOPMENT_TEAM ''" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:8B10BE1726AFFFA6002E1373:buildSettings:PROVISIONING_PROFILE_SPECIFIER ''" $PBXPROJ_PATH -# Release -/usr/libexec/PlistBuddy -c "Set :objects:8B10BE1826AFFFA6002E1373:buildSettings:CODE_SIGN_STYLE 'Manual'" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:8B10BE1826AFFFA6002E1373:buildSettings:DEVELOPMENT_TEAM ''" $PBXPROJ_PATH -/usr/libexec/PlistBuddy -c "Set :objects:8B10BE1826AFFFA6002E1373:buildSettings:PROVISIONING_PROFILE_SPECIFIER ''" $PBXPROJ_PATH - -#修改build number -# Debug -/usr/libexec/PlistBuddy -c "Set :objects:03D13BF72448758C00B599B3:buildSettings:CURRENT_PROJECT_VERSION ${BUILD_NUMBER}" $PBXPROJ_PATH -# Release -/usr/libexec/PlistBuddy -c "Set :objects:03D13BF82448758C00B599B3:buildSettings:CURRENT_PROJECT_VERSION ${BUILD_NUMBER}" $PBXPROJ_PATH - - - - -TARGET_FILE="" -if [ ! -f "Podfile" ];then -TARGET_FILE="${APP_Project}.xcodeproj" -xcodebuild clean -project ${TARGET_FILE} -scheme "${APP_TARGET}" -configuration ${MODE} -xcodebuild CODE_SIGN_STYLE="Manual" -project ${TARGET_FILE} -scheme "${APP_TARGET}" clean CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO -configuration ${MODE} -archivePath ${ArchivePath} archive +if [[ -f Podfile ]]; then + pod install + project_args=(-workspace "${APP_PROJECT}.xcworkspace") else -pod install -TARGET_FILE="${APP_Project}.xcworkspace" -xcodebuild clean -workspace ${TARGET_FILE} -scheme "${APP_TARGET}" -configuration ${MODE} -xcodebuild CODE_SIGN_STYLE="Manual" archive -workspace ${TARGET_FILE} -scheme "${APP_TARGET}" clean CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO -configuration ${MODE} -archivePath ${ArchivePath} -destination 'generic/platform=iOS' + project_args=(-project "${APP_PROJECT}.xcodeproj") fi -xcodebuild -exportArchive -exportOptionsPlist ${Export_Plist_File} -archivePath ${ArchivePath} -exportPath . - -mkdir app -mv *.ipa app && mv *.xcarchive app +xcodebuild \ + "${project_args[@]}" \ + -scheme "${APP_TARGET}" \ + -configuration "${MODE}" \ + -destination 'generic/platform=iOS' \ + CODE_SIGNING_ALLOWED=NO \ + CODE_SIGNING_REQUIRED=NO \ + CODE_SIGN_IDENTITY= \ + DEVELOPMENT_TEAM= \ + PROVISIONING_PROFILE_SPECIFIER= \ + clean build